From d268c8b58ae61c9fe5280a1915a0f17a85e5f1a8 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 18:34:09 +0000 Subject: [PATCH 01/33] feat(azure_ai): add MAI-Image-2.5-Pro image generation pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 13 ++++++++ model_prices_and_context_window.json | 13 ++++++++ .../test_mai_image_generation.py | 32 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 48dded6a323..fff4c2b7e1b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11159,6 +11159,19 @@ ], "deprecation_date": "2026-10-01" }, + "azure_ai/MAI-Image-2.5-Pro": { + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1085, + "output_cost_per_image_token": 0.000106, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-mai-image-2-5-pro-and-mai-voice-2-flash-in-microsoft-foundry/4539446", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 48dded6a323..fff4c2b7e1b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11159,6 +11159,19 @@ ], "deprecation_date": "2026-10-01" }, + "azure_ai/MAI-Image-2.5-Pro": { + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1085, + "output_cost_per_image_token": 0.000106, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-mai-image-2-5-pro-and-mai-voice-2-flash-in-microsoft-foundry/4539446", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index 55656b97c57..27e78d35c69 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -453,6 +453,38 @@ class TestAzureMAIImageGeneration: ) assert round(cost, 10) == round(expected_cost, 10) + def test_mai_image_pro_edit_cost_splits_text_and_image_input(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + model = "azure_ai/MAI-Image-2.5-Pro" + model_info = litellm.get_model_info(model=model, custom_llm_provider="azure_ai") + text_tokens = 37 + image_tokens = 1024 + output_image_tokens = 1024 + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1")], + usage=ImageUsage( + input_tokens=text_tokens + image_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=text_tokens, + image_tokens=image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=text_tokens + image_tokens + output_image_tokens, + ), + ) + + cost = azure_ai_image_cost_calculator(model=model, image_response=image_response) + + expected_cost = ( + text_tokens * model_info["input_cost_per_token"] + + image_tokens * model_info["input_cost_per_image_token"] + + output_image_tokens * model_info["output_cost_per_image_token"] + ) + assert round(cost, 10) == round(expected_cost, 10) + assert model_info["input_cost_per_image_token"] != model_info["input_cost_per_token"] + def test_mai_image_cost_calculator_falls_back_to_flat_image_pricing(self, monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") From c55e9a492441e435c7ff3fe57561a355c96fdfb8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:40:27 +0000 Subject: [PATCH 02/33] registry audit: fireworks/together/openrouter fixes, absorb #28853 #27064 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 87 +++++++++++++++++-- model_prices_and_context_window.json | 87 +++++++++++++++++-- model_prices_and_context_window.schema.json | 4 + 3 files changed, 166 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index accabcf85d3..d3ff6e97c4a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3740,6 +3740,21 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/gpt-image-2": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true + }, "azure_ai/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", @@ -21891,6 +21906,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-coder": { + "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 1.4e-07, "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", @@ -21905,6 +21921,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-r1": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "deepseek", @@ -21960,6 +21977,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-v3.2": { + "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -23678,6 +23696,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "output_cost_per_token_priority": 4.95e-06, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -24064,7 +24101,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -24390,7 +24427,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -41353,6 +41390,7 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v3.2-exp": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2.7e-07, "input_cost_per_token_cache_hit": 2e-08, "litellm_provider": "openrouter", @@ -41374,6 +41412,7 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-r1": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 7e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "openrouter", @@ -46169,8 +46208,8 @@ "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", - "max_input_tokens": 200000, - "max_tokens": 200000, + "max_input_tokens": 202752, + "max_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -46186,8 +46225,8 @@ "deprecation_date": "2026-04-02", "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", - "max_input_tokens": 200000, - "max_tokens": 200000, + "max_input_tokens": 202752, + "max_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -64093,6 +64132,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/glm-5p3": { + "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost_priority": 3.25e-07, + "input_cost_per_token": 1.4e-06, + "input_cost_per_token_priority": 1.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_priority": 5.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast": { "cache_read_input_token_cost": 3.9e-07, "input_cost_per_token": 2.1e-06, @@ -64140,6 +64198,23 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index accabcf85d3..d3ff6e97c4a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3740,6 +3740,21 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/gpt-image-2": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true + }, "azure_ai/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", @@ -21891,6 +21906,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-coder": { + "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 1.4e-07, "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", @@ -21905,6 +21921,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-r1": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "deepseek", @@ -21960,6 +21977,7 @@ "supports_tool_choice": true }, "deepseek/deepseek-v3.2": { + "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -23678,6 +23696,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "output_cost_per_token_priority": 4.95e-06, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -24064,7 +24101,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -24390,7 +24427,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -41353,6 +41390,7 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v3.2-exp": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2.7e-07, "input_cost_per_token_cache_hit": 2e-08, "litellm_provider": "openrouter", @@ -41374,6 +41412,7 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-r1": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 7e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "openrouter", @@ -46169,8 +46208,8 @@ "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", - "max_input_tokens": 200000, - "max_tokens": 200000, + "max_input_tokens": 202752, + "max_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -46186,8 +46225,8 @@ "deprecation_date": "2026-04-02", "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", - "max_input_tokens": 200000, - "max_tokens": 200000, + "max_input_tokens": 202752, + "max_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -64093,6 +64132,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/glm-5p3": { + "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost_priority": 3.25e-07, + "input_cost_per_token": 1.4e-06, + "input_cost_per_token_priority": 1.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_priority": 5.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast": { "cache_read_input_token_cost": 3.9e-07, "input_cost_per_token": 2.1e-06, @@ -64140,6 +64198,23 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://api.fireworks.ai/v1/serverless/models", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 44b2569defd..aaf4d81bcc7 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -137,6 +137,10 @@ "type": "number", "minimum": 0 }, + "cache_read_input_image_token_cost": { + "type": "number", + "minimum": 0 + }, "cache_read_input_token_cost": { "type": "number", "minimum": 0, From 7b3e8afaece0b1aa4f1d9101c11daff3f6ab75c3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:52:56 +0000 Subject: [PATCH 03/33] registry: add cache_read_input_image_token_cost field for azure_ai/gpt-image-2 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/utils.py | 1 + tests/test_litellm/test_utils.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d416e2af33a..de4126a0947 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -254,6 +254,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None cache_read_input_audio_token_cost: ReadOnly[float | None] + cache_read_input_image_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b40c10de428..8986753af3e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -652,6 +652,7 @@ def validate_model_cost_values(model_data, exceptions=None): "cache_creation_input_audio_token_cost", "cache_read_input_token_cost", "cache_read_input_audio_token_cost", + "cache_read_input_image_token_cost", "input_dbu_cost_per_token", "output_db_cost_per_token", "output_dbu_cost_per_token", @@ -740,6 +741,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, From 4a4475fd7046ba0a5eda547aa5a1b14dc717a67d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:03:58 +0000 Subject: [PATCH 04/33] registry: add cache_read_input_image_token_cost to CustomPricingLiteLLMParams denylist Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index de4126a0947..9b78628c726 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3619,6 +3619,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_flex: float | None = None cache_read_input_audio_token_cost: float | None = None + cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None input_cost_per_audio_token: float | None = None input_cost_per_token_cache_hit: float | None = None From a65b0c213660447f064a0bc0bc16653e25843894 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:08:25 +0000 Subject: [PATCH 05/33] registry: regen schema.d.ts for cache_read_input_image_token_cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7aa34c5752c..f300492e902 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30756,6 +30756,8 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Image Token Cost */ + cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 200K Tokens */ @@ -41410,6 +41412,8 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Image Token Cost */ + cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 200K Tokens */ From f9244749e089a1fb0fcd157f5d5d8b2895d65e19 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 06:00:06 +0000 Subject: [PATCH 06/33] fix(proxy): return 422 instead of 429 for BudgetExceededError Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/exceptions.py | 2 +- .../_experimental/mcp_server/auth/user_api_key_auth_mcp.py | 2 +- .../test_litellm/litellm_core_utils/test_litellm_logging.py | 6 +++--- .../mcp_server/auth/test_user_api_key_auth_mcp.py | 6 +++--- .../test_litellm/proxy/auth/test_auth_exception_handler.py | 6 +++--- tests/test_litellm/proxy/auth/test_multi_budget_windows.py | 4 ++-- .../management_endpoints/test_key_management_endpoints.py | 4 ++-- tests/test_litellm/proxy/test_common_request_processing.py | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 14cc16452f0..4b236aec99c 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1002,7 +1002,7 @@ class BudgetExceededError(Exception): ): self.current_cost = current_cost self.max_budget = max_budget - self.status_code = 429 + self.status_code = 422 self.llm_provider = llm_provider or "" self.entity_type = entity_type self.entity_id = entity_id diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index b0d57cb6228..b0640e4f0dd 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1154,7 +1154,7 @@ class MCPRequestHandler: Failures surface with the status the standard pipeline would give them, mirroring ``UserAPIKeyAuthExceptionHandler``: a disallowed route is the route gate's own 403, an - over-budget identity is a 429, a sub-check that raised its own ``HTTPException``/ + over-budget identity is a 422, a sub-check that raised its own ``HTTPException``/ ``ProxyException`` keeps that status, a transient database outage is a retryable 503, and only a genuinely unresolvable failure (a blocked team/project raises a bare ``Exception``, same as the standard pipeline's fallback) becomes the fail-closed 401. Collapsing every diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 626a13c8061..325052ebda9 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3033,7 +3033,7 @@ def test_get_error_information_budget_exceeded_structured_fields(): assert result["error_budget_entity_id"] == "repro-user" assert result["error_budget_limit"] == 1e-06 assert result["error_budget_spend"] == 3.4e-05 - assert result["error_code"] == "429" + assert result["error_code"] == "422" assert result["error_class"] == "BudgetExceededError" assert result["error_rate_limit_type"] == "budget" @@ -6407,7 +6407,7 @@ def test_get_error_information_keeps_traceback_for_unmapped_provider_4xx(): def test_get_error_information_skips_traceback_for_budget_rejection_with_provider(): - """A key-over-budget 429 is the proxy's own rejection even after the auth + """A key-over-budget 422 is the proxy's own rejection even after the auth handler stamps the requested model's provider onto it, so it stays cheap.""" from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -6416,7 +6416,7 @@ def test_get_error_information_skips_traceback_for_budget_rejection_with_provide litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic") ) result = StandardLoggingPayloadSetup.get_error_information(over_budget) - assert result["error_code"] == "429" + assert result["error_code"] == "422" assert result["llm_provider"] == "anthropic" assert result["traceback"] == "" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 4380df194ed..087c5a03498 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -6339,15 +6339,15 @@ class TestMCPDcrBridgeDelegateAdmission: ) return exc_info.value - async def test_over_budget_admission_surfaces_429_not_401(self): - """A validly-authenticated but over-budget identity surfaces the standard pipeline's 429, not + async def test_over_budget_admission_surfaces_422_not_401(self): + """A validly-authenticated but over-budget identity surfaces the standard pipeline's 422, not a misleading 401. Flattening budget to 401 told the caller their credential was invalid, which on a DCR client reads as broken auth and triggers a re-authorize that cannot fix a budget problem. Regression for the status-flattening finding on the live-policy gate.""" import litellm mapped = await self._enforce_with_gate_error(litellm.BudgetExceededError(current_cost=10.0, max_budget=1.0)) - assert mapped.status_code == 429 + assert mapped.status_code == 422 async def test_db_outage_during_policy_surfaces_503_not_401(self): """A transient database outage during the live-policy gate surfaces a retryable 503, not a 401 diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 125b8862dfc..3edc57af124 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -448,7 +448,7 @@ async def test_handle_authentication_error_budget_exceeded(): ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded - assert int(exc_info.value.code) == status.HTTP_429_TOO_MANY_REQUESTS + assert int(exc_info.value.code) == status.HTTP_422_UNPROCESSABLE_CONTENT @pytest.mark.asyncio @@ -687,7 +687,7 @@ def _http_request(client_host: str | None = "10.1.2.3", headers: dict[str, str] {"allow_requests_on_db_unavailable": False}, {}, "10.1.2.3", - id="429_budget_exceeded", + id="422_budget_exceeded", ), ], ) @@ -697,7 +697,7 @@ async def test_auth_failure_logs_requester_ip_address( request_kwargs: dict[str, dict[str, str]], expected_ip: str, ) -> None: - """401s and budget 429s are rejected before `add_litellm_data_to_request` stamps + """401s and budget 422s are rejected before `add_litellm_data_to_request` stamps the caller IP, so without this the failure logs (spend logs, prometheus client_ip) had no IP, and a 401 rarely carries a key or user identity either.""" with ( diff --git a/tests/test_litellm/proxy/auth/test_multi_budget_windows.py b/tests/test_litellm/proxy/auth/test_multi_budget_windows.py index 0f01391b2f5..1c928448bd8 100644 --- a/tests/test_litellm/proxy/auth/test_multi_budget_windows.py +++ b/tests/test_litellm/proxy/auth/test_multi_budget_windows.py @@ -75,7 +75,7 @@ async def test_over_first_window_raises(): await _virtual_key_multi_budget_check(valid_token=token) err = exc_info.value - assert err.status_code == 429 + assert err.status_code == 422 assert "24h" in str(err) assert "Key over" in str(err) @@ -107,7 +107,7 @@ async def test_over_second_window_raises(): await _virtual_key_multi_budget_check(valid_token=token) err = exc_info.value - assert err.status_code == 429 + assert err.status_code == 422 assert "30d" in str(err) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index e2a68988ee2..b02ff47ed52 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -8156,7 +8156,7 @@ async def test_reset_key_spend_resets_budget_windows(monkeypatch): counter without also advancing reset_at is not durable either: the very next request would re-sum the unchanged historical spend and put the counter right back above the window's max_budget, so - _virtual_key_multi_budget_check kept raising BudgetExceededError (429) on + _virtual_key_multi_budget_check kept raising BudgetExceededError (422) on every request even though the key's own reported spend read $0. """ mock_prisma_client = MagicMock() @@ -16593,7 +16593,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch): It used to probe a second, provider-stripped key because the counter was written under the request model instead, which is what let a key report zero - usage while being blocked at 429. + usage while being blocked at 422. """ from unittest.mock import AsyncMock, MagicMock diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index e4ca0b03d59..0b872400be0 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -495,7 +495,7 @@ class TestProxyBaseLLMRequestProcessing: ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded - assert exc_info.value.code == "429" + assert exc_info.value.code == "422" tag_budget_check.assert_awaited_once() _, call_kwargs = tag_budget_check.call_args assert call_kwargs["tags"] == ("guardrail-tag",) @@ -702,7 +702,7 @@ class TestProxyBaseLLMRequestProcessing: ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded - assert exc_info.value.code == "429" + assert exc_info.value.code == "422" assert "guardrail-tag" in exc_info.value.message @pytest.mark.asyncio From 92d3a1d87de637efd57acfd31dd8068f5ee0848c Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 06:10:16 +0000 Subject: [PATCH 07/33] test(proxy): expect 422 for per-model budget rejections on cursor route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/response_api_endpoints/test_endpoints.py | 6 +++--- tests/test_litellm/proxy/test_proxy_server.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index f7abb209015..4153bf7d7ee 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -2099,7 +2099,7 @@ class TestCursorVariantPerModelBudgetEnforcement: response = _post_cursor_with_real_auth(valid_token, attrs, request_model="claude-opus-5-thinking-high") - assert response.status_code == 429, response.text + assert response.status_code == 422, response.text error = response.json()["error"] assert error["type"] == "budget_exceeded" assert "exceeded budget for model=claude-opus-5" in error["message"] @@ -2110,8 +2110,8 @@ class TestCursorVariantPerModelBudgetEnforcement: base_response = _post_cursor_with_real_auth(valid_token, attrs, request_model="claude-opus-5") alias_response = _post_cursor_with_real_auth(valid_token, attrs, request_model="claude-opus-5-fast") - assert base_response.status_code == 429, base_response.text - assert alias_response.status_code == 429, alias_response.text + assert base_response.status_code == 422, base_response.text + assert alias_response.status_code == 422, alias_response.text assert alias_response.json() == base_response.json() diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index f71f9c20f3b..950a6cc3c40 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10872,7 +10872,7 @@ async def test_realtime_session_rejected_in_pre_call_releases_the_budget_reserva """A rate-limit or guardrail rejection happens before route_request, so the relay never runs and no success log can own the reservation. The endpoint must release it on that exit too, or the key stays pinned at the reserved - amount and its next requests 429 with budget_exceeded while /key/info shows + amount and its next requests 422 with budget_exceeded while /key/info shows spend 0 (reproduced live with rpm_limit=1). The client still gets the pre-call error event and the 1011 close it got before.""" reservation: Final = {"reserved_cost": 0.55, "input_cost": 0.0, "finalized": False, "entries": []} From b7e11546b569786bd6733ba7a9c90c54f90984d2 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 06:33:57 +0000 Subject: [PATCH 08/33] test: expect 422 for budget refusals in unification, e2e and integration suites Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/e2e_http.py | 2 +- .../e2e/management/test_key_management_e2e.py | 4 +-- .../budgets/test_budget_enforcement_e2e.py | 30 +++++++++---------- .../budgets/test_multi_window_budget_e2e.py | 2 +- .../test_team_multi_window_budget_e2e.py | 2 +- .../test_partial_update_sequences.py | 4 +-- .../integration/spend/test_cache_and_quota.py | 4 +-- .../test_rate_limit_error_unification.py | 6 ++-- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 4184b6cbefc..d4978601b20 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -95,7 +95,7 @@ class UnauthorizedError(BaseModel): class RateLimitedError(BaseModel): kind: Literal["rate_limited"] = "rate_limited" retry_after_seconds: int | None = None - # litellm overloads 429 for budget_exceeded too, so keep the body to tell them apart. + # keep the body so callers can tell limiter kinds apart. body: str = "" diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index 353b0f7cf09..39a9e657b8c 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -96,8 +96,8 @@ def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None: for _ in range(40): outcome = client.chat_status(key, SPEND_MODEL, f"spend {unique_marker()}") if _is_budget_block(outcome): - assert outcome.status_code == 429, ( - f"budget refusal must be 429, got {outcome.status_code}: {outcome.body[:200]}" + assert outcome.status_code == 422, ( + f"budget refusal must be 422, got {outcome.status_code}: {outcome.body[:200]}" ) return assert outcome.ok, f"paid call failed before the budget tripped ({outcome.status_code}): {outcome.body[:300]}" diff --git a/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py b/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py index 918739863ce..8a9be1d1385 100644 --- a/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py @@ -46,10 +46,10 @@ def _assert_budget_blocks(client: BudgetClient, key: str, *, user: str = "") -> pytest.fail("budget never enforced within the call budget") -def _assert_blocked_429(client: BudgetClient, key: str) -> StreamingResponse: +def _assert_blocked_422(client: BudgetClient, key: str) -> StreamingResponse: blocked = _assert_budget_blocks(client, key) - assert blocked.status_code == 429, ( - f"budget refusal must be 429, got {blocked.status_code}: {blocked.body[:200]}" + assert blocked.status_code == 422, ( + f"budget refusal must be 422, got {blocked.status_code}: {blocked.body[:200]}" ) return blocked @@ -60,7 +60,7 @@ class TestBudgetBlocksPerLevel: key = client.generate_key(max_budget=TINY_CAP) resources.defer(lambda: client.delete_key(key)) - _assert_blocked_429(client, key) + _assert_blocked_422(client, key) @pytest.mark.covers("quota_management.budget.team.blocks_over_limit") def test_team_budget_blocks_every_team_key(self, client: BudgetClient, resources: ResourceManager) -> None: @@ -71,10 +71,10 @@ class TestBudgetBlocksPerLevel: sibling_key = client.generate_key(team_id=team_id) resources.defer(lambda: client.delete_key(sibling_key)) - _assert_blocked_429(client, spender_key) + _assert_blocked_422(client, spender_key) sibling = _chat(client, sibling_key) - assert is_budget_block(sibling) and sibling.status_code == 429, ( - f"a sibling key on the capped team must get the same 429 budget_exceeded, " + assert is_budget_block(sibling) and sibling.status_code == 422, ( + f"a sibling key on the capped team must get the same 422 budget_exceeded, " f"got {sibling.status_code}: {sibling.body[:200]}" ) @@ -99,10 +99,10 @@ class TestBudgetBlocksPerLevel: team_key = client.generate_key(team_id=team_id, user_id=user_id) resources.defer(lambda: client.delete_key(team_key)) - _assert_blocked_429(client, first_key) + _assert_blocked_422(client, first_key) second = _chat(client, second_key) - assert is_budget_block(second) and second.status_code == 429, ( - f"the second personal key of a user over budget must get the same 429 budget_exceeded, " + assert is_budget_block(second) and second.status_code == 422, ( + f"the second personal key of a user over budget must get the same 422 budget_exceeded, " f"got {second.status_code}: {second.body[:200]}" ) team_result = _chat(client, team_key) @@ -133,7 +133,7 @@ class TestBudgetBlocksPerLevel: key = client.generate_key(team_id=team_id) resources.defer(lambda: client.delete_key(key)) - blocked = _assert_blocked_429(client, key) + blocked = _assert_blocked_422(client, key) assert f"Organization={org_id}" in blocked.body, ( f"refusal must name the org as the blocker, got: {blocked.body[:200]}" ) @@ -155,7 +155,7 @@ class TestBudgetBlocksPerLevel: teammate_key = client.generate_key(team_id=team_id, user_id=teammate_id) resources.defer(lambda: client.delete_key(teammate_key)) - _assert_blocked_429(client, member_key) + _assert_blocked_422(client, member_key) require_successful_call(_chat(client, teammate_key)) @@ -176,7 +176,7 @@ class TestKeyBudgetBlocksAcrossKeyKinds: control_key = client.generate_key(user_id=user_id) resources.defer(lambda: client.delete_key(control_key)) - _assert_blocked_429(client, capped_key) + _assert_blocked_422(client, capped_key) require_successful_call(_chat(client, control_key)) @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") @@ -188,7 +188,7 @@ class TestKeyBudgetBlocksAcrossKeyKinds: control_key = client.generate_key(team_id=team_id) resources.defer(lambda: client.delete_key(control_key)) - _assert_blocked_429(client, capped_key) + _assert_blocked_422(client, capped_key) require_successful_call(_chat(client, control_key)) @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") @@ -205,5 +205,5 @@ class TestKeyBudgetBlocksAcrossKeyKinds: control_key = client.generate_key(team_id=team_id, user_id=member_id) resources.defer(lambda: client.delete_key(control_key)) - _assert_blocked_429(client, capped_key) + _assert_blocked_422(client, capped_key) require_successful_call(_chat(client, control_key)) diff --git a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py index e1cca0c0414..e04f857545d 100644 --- a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py @@ -102,7 +102,7 @@ def test_long_window_blocks_after_short_window_resets(client: BudgetClient, reso # 1. drive the key to get blocked by SHORT_WINDOW, assert it's budget error blocked = _drive_to_block(client, key) - assert blocked.status_code == 429, f"budget block was not a 429: {blocked.status_code} {blocked.body[:200]}" + assert blocked.status_code == 422, f"budget block was not a 422: {blocked.status_code} {blocked.body[:200]}" # 2. check the reset times of both budget windows after we drove to being blocked blocked_reset_at = window_reset_at(client.key_budget_windows(key), SHORT_WINDOW) diff --git a/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py b/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py index 1db68e6afe9..7683132776b 100644 --- a/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py @@ -101,7 +101,7 @@ def test_team_long_window_blocks_after_short_window_resets(client: BudgetClient, # 1. drive the key to being blocked, assert its blocked by budget budget_exceeded blocked = _drive_to_block(client, key) - assert blocked.status_code == 429, f"budget block was not a 429: {blocked.status_code} {blocked.body[:200]}" + assert blocked.status_code == 422, f"budget block was not a 422: {blocked.status_code} {blocked.body[:200]}" # 2. check the the teams budget windows blocked_reset_at = window_reset_at(client.team_budget_windows(team_id), SHORT_WINDOW) diff --git a/tests/integration/management/test_partial_update_sequences.py b/tests/integration/management/test_partial_update_sequences.py index d79c145a685..64d807de39d 100644 --- a/tests/integration/management/test_partial_update_sequences.py +++ b/tests/integration/management/test_partial_update_sequences.py @@ -97,7 +97,7 @@ def test_zero_false_and_empty_values_are_not_treated_as_omission(gateway: Gatewa "POST", "/v1/chat/completions", {"model": models[0], "messages": [{"role": "user", "content": "zero budget"}]}, key=key, ) - assert denied.status_code == 429, denied.text + assert denied.status_code == 422, denied.text assert denied.json()["error"]["type"] == "budget_exceeded" gateway.post("/key/update", {"key": key, "max_budget": 1, "models": [], "metadata": {}}) info: Final = object_value(gateway.get("/key/info", {"key": key})["info"]) @@ -127,7 +127,7 @@ def test_zero_false_and_empty_values_are_not_treated_as_omission(gateway: Gatewa "POST", "/v1/chat/completions", {"model": models[0], "messages": [{"role": "user", "content": "updated zero budget"}]}, key=key, ) - assert zero_after_update.status_code == 429, zero_after_update.text + assert zero_after_update.status_code == 422, zero_after_update.text assert zero_after_update.json()["error"]["type"] == "budget_exceeded" gateway.post("/key/update", {"key": key, "max_budget": None}) assert read_rows( diff --git a/tests/integration/spend/test_cache_and_quota.py b/tests/integration/spend/test_cache_and_quota.py index 840594c1a96..d32297765f6 100644 --- a/tests/integration/spend/test_cache_and_quota.py +++ b/tests/integration/spend/test_cache_and_quota.py @@ -185,7 +185,7 @@ def test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores(gat {"model": model, "messages": [{"role": "user", "content": f"over budget {uuid.uuid4().hex}"}]}, key=key, ) - assert denied.status_code == 429 and denied.json()["error"]["type"] == "budget_exceeded", denied.text + assert denied.status_code == 422 and denied.json()["error"]["type"] == "budget_exceeded", denied.text assert upstream.get("/__observations").json()["requests"] == [] assert gateway.chat(model, key=control, text=f"control {uuid.uuid4().hex}")["usage"]["total_tokens"] == 40 gateway.post("/key/update", {"key": key, "spend": 0}) @@ -205,7 +205,7 @@ def test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores(gat {"model": model, "messages": [{"role": "user", "content": f"boundary again {uuid.uuid4().hex}"}]}, key=key, ) - assert denied_again.status_code == 429 and denied_again.json()["error"]["type"] == "budget_exceeded", ( + assert denied_again.status_code == 422 and denied_again.json()["error"]["type"] == "budget_exceeded", ( denied_again.text ) assert upstream.get("/__observations").json()["requests"] == [] diff --git a/tests/test_litellm/test_rate_limit_error_unification.py b/tests/test_litellm/test_rate_limit_error_unification.py index 8241b29aff1..256d33845a5 100644 --- a/tests/test_litellm/test_rate_limit_error_unification.py +++ b/tests/test_litellm/test_rate_limit_error_unification.py @@ -1397,10 +1397,10 @@ class TestBudgetExceededErrorSurfacesUnifiedFields: assert e.llm_provider == "anthropic" def test_should_keep_existing_status_code_and_message(self): - # Backward-compat guard: existing callers depend on `status_code=429` + # Backward-compat guard: existing callers depend on `status_code=422` # and the canonical message format. e = litellm.BudgetExceededError(current_cost=0.000109, max_budget=0.0001) - assert e.status_code == 429 + assert e.status_code == 422 assert "Current cost: 0.000109" in e.message assert "Max budget: 0.0001" in e.message @@ -1424,7 +1424,7 @@ class TestBudgetExceededErrorSurfacesUnifiedFields: info = StandardLoggingPayloadSetup.get_error_information(e) assert info["error_rate_limit_category"] == "litellm_rate_limit" assert info["error_rate_limit_type"] == "budget" - assert info["error_code"] == "429" + assert info["error_code"] == "422" assert info["error_class"] == "BudgetExceededError" def test_should_propagate_llm_provider_to_standard_logging_payload(self): From bf804f51885820a6163ddcd2322ad4494f1caeb8 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 07:00:10 +0000 Subject: [PATCH 09/33] feat(proxy): add budget_exceeded_status_code setting to restore 429 for budget refusals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/exceptions.py | 3 ++- tests/test_litellm/test_rate_limit_error_unification.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 738dd0cac76..be8f59d210b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -400,6 +400,7 @@ default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers +budget_exceeded_status_code: int = 422 # set to 429 to restore the pre-422 budget_exceeded response code budget_duration: Optional[str] = ( None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). ) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 4b236aec99c..c8de2ab12ed 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -16,6 +16,7 @@ from typing import Any, Final import httpx import openai +import litellm from litellm.types.utils import LiteLLMCommonStrings from litellm.types.vector_stores import VectorStoreSearchFailure @@ -1002,7 +1003,7 @@ class BudgetExceededError(Exception): ): self.current_cost = current_cost self.max_budget = max_budget - self.status_code = 422 + self.status_code = litellm.budget_exceeded_status_code self.llm_provider = llm_provider or "" self.entity_type = entity_type self.entity_id = entity_id diff --git a/tests/test_litellm/test_rate_limit_error_unification.py b/tests/test_litellm/test_rate_limit_error_unification.py index 256d33845a5..e5acba938c7 100644 --- a/tests/test_litellm/test_rate_limit_error_unification.py +++ b/tests/test_litellm/test_rate_limit_error_unification.py @@ -1404,6 +1404,11 @@ class TestBudgetExceededErrorSurfacesUnifiedFields: assert "Current cost: 0.000109" in e.message assert "Max budget: 0.0001" in e.message + def test_should_honor_budget_exceeded_status_code_override(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "budget_exceeded_status_code", 429) + e = litellm.BudgetExceededError(current_cost=0.5, max_budget=0.1) + assert e.status_code == 429 + def test_should_still_be_catchable_as_exception_not_rate_limit_error(self): # Critical: we deliberately did NOT make BudgetExceededError a # RateLimitError subclass. Existing `except BudgetExceededError:` From 3ebd5add3230f0acb01fb134434c55bbda4e4104 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 17:20:16 +0000 Subject: [PATCH 10/33] fix(anthropic): forward safeguards and anthropic-beta unchanged on native /v1/messages Native Anthropic Messages requests derived their allowlist from AnthropicMessagesRequestOptionalParams, which lacked safeguards, and the shared beta-header filter dropped betas unknown to the provider mapping even when the upstream is api.anthropic.com itself. Claude Code auto mode then saw no safeguard_results and fell back to billed classifier calls Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/transformation.py | 4 + litellm/types/llms/anthropic.py | 1 + .../anthropic_messages/anthropic_response.py | 1 + ...erimental_pass_through_messages_handler.py | 108 ++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 5fa686b7560..eed30c2698c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -79,10 +79,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "speed", "output_config", "reasoning_effort", + "safeguards", # TODO: Add Anthropic `metadata` support # "metadata", ] + def should_filter_anthropic_beta_headers(self) -> bool: + return self._resolved_provider != "anthropic" + def _remove_scope_from_cache_control(self, anthropic_messages_request: dict) -> None: """ Remove `scope` field from cache_control blocks. diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index bcd24695f25..f4fe7a0bf14 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -411,6 +411,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): output_config: AnthropicOutputConfig | None # Configuration for Claude's output behavior cache_control: dict[str, Any] | None # Automatic prompt caching reasoning_effort: str | None + safeguards: ReadOnly[dict[str, object] | None] class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 038a23a3ca2..41060e96d85 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -97,3 +97,4 @@ class AnthropicMessagesResponse(TypedDict, total=False): type: Literal["message"] | None usage: AnthropicUsage | None context_management: NotRequired[ContextManagementResponse] + safeguard_results: NotRequired[ReadOnly[dict[str, object]]] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 997a97c6fd3..4246e70bbbf 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1438,3 +1438,111 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped(): ) assert "Traceback" not in str(excinfo.value) + + +@pytest.mark.asyncio +async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): + """Regression test for LIT-8232. Claude Code auto mode sends a `safeguards` body + field paired with a beta value the gateway has never seen. Both must reach + api.anthropic.com unchanged or the session falls back to billed classifier calls.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} + client_betas = "safeguards-2026-09-01,interleaved-thinking-2025-05-14" + captured: dict[str, object] = {} + + def upstream_records_the_request(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + captured["anthropic-beta"] = request.headers.get("anthropic-beta") + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + "safeguard_results": {"verdict": "allow"}, + }, + request=request, + ) + + upstream = AsyncHTTPHandler() + upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_records_the_request)) + + response = await handler.anthropic_messages( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model="anthropic/claude-haiku-4-5", + custom_llm_provider="anthropic", + api_key="sk-test", + client=upstream, + safeguards=safeguards, + extra_headers={"anthropic-beta": client_betas}, + ) + + assert captured["body"]["safeguards"] == safeguards + assert set(captured["anthropic-beta"].split(",")) == set(client_betas.split(",")) + assert response["safeguard_results"] == {"verdict": "allow"} + + +@pytest.mark.asyncio +async def test_anthropic_messages_streaming_forwards_safeguards_and_keeps_safeguard_results(): + """Streaming sibling of the LIT-8232 regression: the request must still carry + `safeguards` and the `safeguard_results` Anthropic emits on `message_start` and + `message_delta` must reach the client byte for byte.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} + safeguard_results = {"verdict": "allow", "checks": ["shell_command"]} + captured: dict[str, object] = {} + message_start = { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + "safeguard_results": safeguard_results, + }, + } + message_delta = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None, "safeguard_results": safeguard_results}, + "usage": {"output_tokens": 1}, + } + sse = "".join( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" + for event in (message_start, message_delta, {"type": "message_stop"}) + ) + + def upstream_streams_safeguard_results(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse.encode(), request=request) + + upstream = AsyncHTTPHandler() + upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_streams_safeguard_results)) + + stream = await handler.anthropic_messages( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model="anthropic/claude-haiku-4-5", + custom_llm_provider="anthropic", + api_key="sk-test", + client=upstream, + stream=True, + safeguards=safeguards, + ) + raw = b"".join([chunk async for chunk in stream]).decode() + events = [json.loads(line[len("data: ") :]) for line in raw.splitlines() if line.startswith("data: ")] + + assert captured["body"]["safeguards"] == safeguards + assert events[0]["message"]["safeguard_results"] == safeguard_results + assert [e for e in events if e["type"] == "message_delta"][0]["delta"]["safeguard_results"] == safeguard_results From b59028d525352454de672621c2b223c5d75e57b1 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 20 Sep 2026 17:37:56 +0000 Subject: [PATCH 11/33] fix(anthropic): strip safeguards on the adapter path and type it on streaming chunks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../experimental_pass_through/adapters/handler.py | 2 +- litellm/types/llms/anthropic.py | 2 ++ .../test_handler_output_config_passthrough.py | 13 +++++++++++++ ...ic_experimental_pass_through_messages_handler.py | 6 ------ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 87a29ca50ba..54d10837d74 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: from litellm.router import Router # Anthropic-only keys already mapped by the translator; strip on extra_kwargs re-merge. -ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"output_config"}) +ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"output_config", "safeguards"}) _AnthropicMessages: TypeAlias = "list[dict[str, object]]" _AnthropicSystem: TypeAlias = "str | list[dict[str, object]] | None" diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f4fe7a0bf14..f57591d0262 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -531,6 +531,7 @@ class AnthropicStopDetails(TypedDict, total=False): class MessageDelta(TypedDict, total=False): stop_reason: str | None stop_details: ReadOnly[AnthropicStopDetails] + safeguard_results: ReadOnly[dict[str, object]] class ServerToolUsage(TypedDict, total=False): @@ -601,6 +602,7 @@ class MessageChunk(TypedDict, total=False): stop_reason: str | None stop_sequence: str | None usage: UsageDelta + safeguard_results: ReadOnly[dict[str, object]] class MessageStartBlock(TypedDict): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py index a944afc6152..d6de6372e0b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py @@ -110,6 +110,19 @@ class TestOutputConfigStrippedFromCompletionKwargs: "reject it with 400 'Extra inputs are not permitted'" ) + def test_safeguards_is_stripped_for_non_anthropic_target(self): + extra_kwargs = { + "custom_llm_provider": "azure", + "safeguards": {"auto_mode": {"enabled": True, "version": "2026-09-01"}}, + } + + result = _call_prepare(extra_kwargs=extra_kwargs) + + completion_kwargs = result[0] if isinstance(result, tuple) else result + assert "safeguards" not in completion_kwargs, ( + "safeguards is an Anthropic-only field; OpenAI-format backends reject it with 400" + ) + def test_output_config_format_translated_to_response_format(self): """When ``output_config`` carries structured-output ``format``, the translator now maps it to OpenAI's ``response_format`` so non-Anthropic diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 4246e70bbbf..0acb9d634a3 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1442,9 +1442,6 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped(): @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): - """Regression test for LIT-8232. Claude Code auto mode sends a `safeguards` body - field paired with a beta value the gateway has never seen. Both must reach - api.anthropic.com unchanged or the session falls back to billed classifier calls.""" from litellm.llms.anthropic.experimental_pass_through.messages import handler safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} @@ -1491,9 +1488,6 @@ async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthro @pytest.mark.asyncio async def test_anthropic_messages_streaming_forwards_safeguards_and_keeps_safeguard_results(): - """Streaming sibling of the LIT-8232 regression: the request must still carry - `safeguards` and the `safeguard_results` Anthropic emits on `message_start` and - `message_delta` must reach the client byte for byte.""" from litellm.llms.anthropic.experimental_pass_through.messages import handler safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} From 83223885e6d4f9dd536f73214b446e1f14b64244 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:09:59 +0000 Subject: [PATCH 12/33] fix(responses): forward safety_identifier through the chat completion bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 1 + tests/e2e/llm_translation/endpoints_client.py | 3 + .../e2e/llm_translation/test_responses_e2e.py | 89 ++++++++++++++++++- tests/e2e/models.py | 2 + tests/e2e/provider_edge.py | 10 ++- .../test_litellm_completion_responses.py | 10 +++ 6 files changed, 109 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index cf3075ee28d..3ca2cc28c9a 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -454,6 +454,7 @@ class LiteLLMCompletionResponsesConfig: "stream": stream, "metadata": kwargs.get("metadata"), "service_tier": kwargs.get("service_tier"), + "safety_identifier": responses_api_request.get("safety_identifier"), "web_search_options": web_search_options, "response_format": response_format, "reasoning_effort": reasoning.effort, diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index 4d2c73e7078..eb7bae2220c 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -75,6 +75,7 @@ class ResponsesRequest(BaseModel): stream: bool = False tools: list[ResponsesFunctionTool] | None = None guardrails: list[str] | None = None + safety_identifier: str | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -316,6 +317,7 @@ class EndpointsClient: *, stream: bool = False, guardrails: list[str] | None = None, + safety_identifier: str | None = None, ) -> StreamingResponse: return self._send( "/v1/responses", @@ -326,6 +328,7 @@ class EndpointsClient: instructions="You are a helpful assistant", stream=stream, guardrails=guardrails, + safety_identifier=safety_identifier, ), stream=stream, ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 3fcf2d1ac05..083bc66b3f2 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -8,13 +8,18 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations import json -from typing import cast +import threading +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, cast import pytest -from e2e_config import unique_marker +from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import ( assert_client_error, require_successful_call, + unwrap, ) from endpoints_client import ( EndpointsClient, @@ -26,7 +31,9 @@ from endpoints_client import ( ResponsesStreamEventType, ) from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import ChatBody, ChatMessage, LiteLLMParamsBody +from provider_edge import LiveEdge, start_provider_edge +from provider_edge_bedrock import bedrock_signer from pydantic import BaseModel, ValidationError pytestmark = pytest.mark.e2e @@ -39,6 +46,33 @@ class _OptionalResponsesBody(BaseModel): BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +BEDROCK_EDGE_REGION: Final = "us-east-1" +BEDROCK_EDGE_MOUNT: Final = f"bedrock/{BEDROCK_EDGE_REGION}" + + +class ConverseRequestBody(BaseModel): + additionalModelRequestFields: dict[str, str] | None = None + + +@dataclass(slots=True) +class ConverseRequestCapture: + """The Converse bodies the proxy actually sent upstream, as seen by a live + edge sitting between the proxy and Bedrock.""" + + _bodies: list[ConverseRequestBody] = field(default_factory=list) + _lock: threading.Lock = field(default_factory=threading.Lock) + + def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None: + if body is None or "/converse" not in url: + return + with self._lock: + self._bodies.append(ConverseRequestBody.model_validate_json(body)) + + @property + def bodies(self) -> tuple[ConverseRequestBody, ...]: + with self._lock: + return tuple(self._bodies) + WEATHER_TOOL = ResponsesFunctionTool( name="get_weather", @@ -295,6 +329,55 @@ class TestResponses: arguments = WeatherArguments.model_validate(raw_arguments) assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + @pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"]) + def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( + self, endpoints_client: EndpointsClient, resources: ResourceManager, endpoint: str + ) -> None: + capture: Final = ConverseRequestCapture() + edge: Final = start_provider_edge( + LiveEdge(observe_request=capture.observe, sign=bedrock_signer(BEDROCK_EDGE_REGION)), + mounts=MappingProxyType({BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}), + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + ) + resources.defer(edge.shutdown) + model: Final = f"e2e-responses-{unique_marker()}" + model_id: Final = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model=BEDROCK_CONVERSE_BACKEND, + api_base=edge.edge.api_base(BEDROCK_EDGE_MOUNT), + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name=BEDROCK_EDGE_REGION, + allowed_openai_params=["safety_identifier"], + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key: Final = resources.key() + safety_identifier: Final = f"end-user-{unique_marker()}" + + if endpoint == "/v1/responses": + responses_result: Final = endpoints_client.responses( + key, model, "reply with one word", safety_identifier=safety_identifier + ) + require_successful_call(responses_result) + else: + unwrap( + endpoints_client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="reply with one word")], + safety_identifier=safety_identifier, + ), + ) + ) + + assert [body.additionalModelRequestFields for body in capture.bodies] == [ + {"safety_identifier": safety_identifier} + ], f"{endpoint} did not forward safety_identifier to Bedrock Converse: {capture.bodies}" + @pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400") @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") def test_missing_input_returns_error( diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 4b202e3c663..47ef672ebec 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -298,6 +298,7 @@ class ChatBody(BaseModel): max_completion_tokens: int | None = None temperature: float | None = None user: str | None = None + safety_identifier: str | None = None metadata: ChatMetadata | None = None reasoning_effort: str | None = None thinking: ThinkingParam | None = None @@ -976,6 +977,7 @@ class LiteLLMParamsBody(BaseModel): api_base: str | None = None api_version: str | None = None realtime_protocol: str | None = None + allowed_openai_params: list[str] | None = None aws_access_key_id: str | None = None aws_secret_access_key: str | None = None aws_region_name: str | None = None diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index 136b00208f7..fc10dde2a77 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -99,6 +99,7 @@ from provider_cache import ( SIGNATURE_HEADERS, CacheEdge, MountPolicy, + RequestSigner, is_bedrock, scoped_edge_base, split_test_segment, @@ -539,6 +540,7 @@ class ReplayEdge: @dataclass(frozen=True, slots=True) class LiveEdge: observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None + sign: RequestSigner | None = None type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge | CacheEdge @@ -788,14 +790,16 @@ def _handle_live( method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float, cache: CacheEdge | None = None, mount: str = "", test_key: str | None = None, observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None, + sign: RequestSigner | None = None, ) -> EdgeOutcome: forwarded: Final = { name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS } if observe_request is not None: observe_request(url, forwarded, body) + outbound: Final = forwarded if sign is None else sign(method, url, forwarded, body) head: Final = ( - forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) + forward_stream(method, url, headers=outbound, body=body, timeout=timeout) if cache is None else cache.forward(mount, method, url, forwarded, body, timeout, test_key=test_key) ) match head: @@ -871,10 +875,10 @@ def handle_edge_request( method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, backend, mount, test_key, ) - case LiveEdge(observe_request=observe_request): + case LiveEdge(observe_request=observe_request, sign=sign): return _handle_live( method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, - observe_request=observe_request, + observe_request=observe_request, sign=sign, ) case RecordEdge(): return _handle_record( diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 6077f281a81..7b9de4644b4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1248,6 +1248,16 @@ class TestFunctionCallTransformation: assert "tool_choice" not in result assert "tools" not in result + def test_safety_identifier_forwarded_to_chat_completion_request(self) -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="bedrock/global.openai.gpt-5.6-luna", + input="hi", + responses_api_request={"safety_identifier": "user-7f3a"}, + custom_llm_provider="bedrock", + ) + + assert result["safety_identifier"] == "user-7f3a" + def test_parallel_tool_calls_dropped_when_no_chat_tools_remain(self) -> None: transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request codex_tool_search: Final = { From ba93c7402a029ac7efb41148e31b3a6b1d186f8b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:53:10 +0000 Subject: [PATCH 13/33] test(e2e): tolerate provider retries in safety_identifier capture assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/llm_translation/test_responses_e2e.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 083bc66b3f2..a5dc48a015f 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -374,9 +374,11 @@ class TestResponses: ) ) - assert [body.additionalModelRequestFields for body in capture.bodies] == [ - {"safety_identifier": safety_identifier} - ], f"{endpoint} did not forward safety_identifier to Bedrock Converse: {capture.bodies}" + forwarded: Final = tuple(body.additionalModelRequestFields for body in capture.bodies) + assert forwarded, f"{endpoint} produced no Bedrock Converse request" + assert forwarded == ({"safety_identifier": safety_identifier},) * len(forwarded), ( + f"{endpoint} did not forward safety_identifier to Bedrock Converse on every attempt: {capture.bodies}" + ) @pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400") @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") From 7e0fa40fe3b260efe8447077fa4726e8e0080729 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:17:03 +0000 Subject: [PATCH 14/33] test(e2e): gate the Bedrock edge capture behind a provider_edge_host opt-in The Buildkite ephemeral stack runs the gateway in another pod, so it cannot reach the pytest host's provider edge. The GitHub changed-e2e lane runs gateways on the runner and sets E2E_PROVIDER_EDGE_HOST_REACHABLE Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-e2e-changed.yml | 1 + tests/e2e/AGENTS.md | 2 +- tests/e2e/conftest.py | 7 +++++++ tests/e2e/e2e_config.py | 1 + tests/e2e/llm_translation/test_responses_e2e.py | 1 + tests/e2e/pytest.ini | 1 + 6 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index c9f08deb36e..f659afad7e3 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -175,6 +175,7 @@ jobs: env: TESTS: ${{ needs.detect.outputs.tests }} E2E_FIXTURE_MODE: live + E2E_PROVIDER_EDGE_HOST_REACHABLE: '1' run: | umask 077 read -r -a test_files <<< "${TESTS}" diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 78ba2ec4ed1..c25e958242f 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -122,7 +122,7 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py` -Current limits: Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode +Current limits: Bedrock cannot be mounted in record or replay (SigV4 signs the Host header, so a rewritten api_base fails signature verification); a test that needs to observe the Converse body registers its own `LiveEdge` with `provider_edge_bedrock.bedrock_signer` re-signing the forwarded request, and carries the `provider_edge_host` opt-in marker because the gateway must reach the pytest host, which the Buildkite ephemeral stack cannot (the GitHub changed-e2e lane, whose gateways run on the runner, sets `E2E_PROVIDER_EDGE_HOST_REACHABLE`). Deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode ## Typing diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 52a634693c5..8776d00d502 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -31,6 +31,7 @@ from e2e_config import ( MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, + PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV, @@ -59,6 +60,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "redis_chaos": REDIS_CHAOS_OPT_IN_ENV, "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, + "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, } ) @@ -143,6 +145,11 @@ def pytest_configure(config: pytest.Config) -> None: "mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless " "E2E_MCP_OAUTH_LIVE is set", ) + config.addinivalue_line( + "markers", + "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " + "gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index a79c158f9c4..14de4619664 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -146,6 +146,7 @@ PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" +PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index a5dc48a015f..1c2336bd166 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -329,6 +329,7 @@ class TestResponses: arguments = WeatherArguments.model_validate(raw_arguments) assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + @pytest.mark.provider_edge_host @pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"]) def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( self, endpoints_client: EndpointsClient, resources: ResourceManager, endpoint: str diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 97acb9ec52b..c6acd449884 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -13,3 +13,4 @@ markers = cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set + provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set From 333fadad6cebe500ff68f702fcb9d9028ad2c6f2 Mon Sep 17 00:00:00 2001 From: yassin Date: Mon, 21 Sep 2026 06:25:43 +0000 Subject: [PATCH 15/33] fix(helm): render a fixed replicaCount on componentized deployments when HPA is disabled Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm/templates/backend/deployment.yaml | 3 + .../litellm/templates/gateway/deployment.yaml | 3 + helm/litellm/templates/ui/deployment.yaml | 3 + helm/litellm/tests/replica_count_tests.yaml | 84 +++++++++++++++++++ helm/litellm/values.yaml | 8 ++ 5 files changed, 101 insertions(+) create mode 100644 helm/litellm/tests/replica_count_tests.yaml diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 0db2f0b3d43..6370f581d71 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -7,6 +7,9 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend spec: + {{- if not .Values.backend.hpa.enabled }} + replicas: {{ .Values.backend.replicaCount }} + {{- end }} {{- with .Values.backend.strategy }} strategy: {{- toYaml . | nindent 4 }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index c06cc9583a0..0c537b1bdeb 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -7,6 +7,9 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: gateway spec: + {{- if not .Values.gateway.hpa.enabled }} + replicas: {{ .Values.gateway.replicaCount }} + {{- end }} {{- with .Values.gateway.strategy }} strategy: {{- toYaml . | nindent 4 }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index b992b347bad..b794418b7e9 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -7,6 +7,9 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui spec: + {{- if not .Values.ui.hpa.enabled }} + replicas: {{ .Values.ui.replicaCount }} + {{- end }} {{- with .Values.ui.strategy }} strategy: {{- toYaml . | nindent 4 }} diff --git a/helm/litellm/tests/replica_count_tests.yaml b/helm/litellm/tests/replica_count_tests.yaml new file mode 100644 index 00000000000..e6a28689ff5 --- /dev/null +++ b/helm/litellm/tests/replica_count_tests.yaml @@ -0,0 +1,84 @@ +suite: test fixed replica count when HPA is disabled +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - ui/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: gateway renders replicaCount into spec.replicas when its HPA is disabled + template: gateway/deployment.yaml + set: + gateway.hpa.enabled: false + gateway.replicaCount: 3 + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 3 + + - it: backend renders replicaCount into spec.replicas when its HPA is disabled + template: backend/deployment.yaml + set: + backend.hpa.enabled: false + backend.replicaCount: 2 + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: ui renders replicaCount into spec.replicas when its HPA is disabled + template: ui/deployment.yaml + set: + ui.hpa.enabled: false + ui.replicaCount: 2 + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: replicaCount 0 scales the gateway to zero instead of being treated as unset + template: gateway/deployment.yaml + set: + gateway.hpa.enabled: false + gateway.replicaCount: 0 + asserts: + - equal: + path: spec.replicas + value: 0 + + - it: every component omits spec.replicas when its HPA is enabled, so the autoscaler owns the count + set: + gateway.hpa.enabled: true + gateway.replicaCount: 3 + backend.hpa.enabled: true + backend.replicaCount: 3 + ui.hpa.enabled: true + ui.replicaCount: 3 + asserts: + - notExists: + path: spec.replicas + template: gateway/deployment.yaml + - notExists: + path: spec.replicas + template: backend/deployment.yaml + - notExists: + path: spec.replicas + template: ui/deployment.yaml + + - it: a component with HPA disabled renders replicas while a sibling with HPA enabled does not + set: + gateway.hpa.enabled: false + gateway.replicaCount: 4 + backend.hpa.enabled: true + backend.replicaCount: 4 + asserts: + - equal: + path: spec.replicas + value: 4 + template: gateway/deployment.yaml + - notExists: + path: spec.replicas + template: backend/deployment.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 4ca54131d6a..d7836b03a34 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -397,6 +397,10 @@ gateway: # failureThreshold: 30 # periodSeconds: 10 startupProbe: {} + # Fixed pod count, rendered into the Deployment's spec.replicas only when + # hpa.enabled is false. With the HPA on, the autoscaler owns the count and + # this value is ignored. + replicaCount: 1 hpa: enabled: true minReplicas: 1 @@ -524,6 +528,8 @@ backend: strategy: {} # Optional startupProbe; same shape as gateway.startupProbe. Empty by default. startupProbe: {} + # Same semantics as gateway.replicaCount. + replicaCount: 1 hpa: enabled: true minReplicas: 1 @@ -590,6 +596,8 @@ ui: strategy: {} # Optional startupProbe; same shape as gateway.startupProbe. Empty by default. startupProbe: {} + # Same semantics as gateway.replicaCount. + replicaCount: 1 hpa: enabled: false minReplicas: 1 From d266d7324b49099ba782a90669cf8d6d1d2d1e54 Mon Sep 17 00:00:00 2001 From: yassin Date: Mon, 21 Sep 2026 06:35:05 +0000 Subject: [PATCH 16/33] fix(helm): leave spec.replicas unset unless replicaCount is explicitly configured Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm/templates/backend/deployment.yaml | 2 +- helm/litellm/templates/gateway/deployment.yaml | 2 +- helm/litellm/templates/ui/deployment.yaml | 2 +- helm/litellm/tests/replica_count_tests.yaml | 16 ++++++++++++++++ helm/litellm/values.yaml | 13 +++++++------ 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 6370f581d71..3eb64e5528c 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -7,7 +7,7 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend spec: - {{- if not .Values.backend.hpa.enabled }} + {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }} replicas: {{ .Values.backend.replicaCount }} {{- end }} {{- with .Values.backend.strategy }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 0c537b1bdeb..49b452b3053 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -7,7 +7,7 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: gateway spec: - {{- if not .Values.gateway.hpa.enabled }} + {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }} replicas: {{ .Values.gateway.replicaCount }} {{- end }} {{- with .Values.gateway.strategy }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index b794418b7e9..efee2d5fc34 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -7,7 +7,7 @@ metadata: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui spec: - {{- if not .Values.ui.hpa.enabled }} + {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }} replicas: {{ .Values.ui.replicaCount }} {{- end }} {{- with .Values.ui.strategy }} diff --git a/helm/litellm/tests/replica_count_tests.yaml b/helm/litellm/tests/replica_count_tests.yaml index e6a28689ff5..791e47ff798 100644 --- a/helm/litellm/tests/replica_count_tests.yaml +++ b/helm/litellm/tests/replica_count_tests.yaml @@ -49,6 +49,22 @@ tests: path: spec.replicas value: 0 + - it: a component with HPA disabled but no replicaCount set keeps omitting spec.replicas, so upgrades do not reset a hand-scaled Deployment + set: + gateway.hpa.enabled: false + backend.hpa.enabled: false + ui.hpa.enabled: false + asserts: + - notExists: + path: spec.replicas + template: gateway/deployment.yaml + - notExists: + path: spec.replicas + template: backend/deployment.yaml + - notExists: + path: spec.replicas + template: ui/deployment.yaml + - it: every component omits spec.replicas when its HPA is enabled, so the autoscaler owns the count set: gateway.hpa.enabled: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index d7836b03a34..2c0c7151a32 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -397,10 +397,11 @@ gateway: # failureThreshold: 30 # periodSeconds: 10 startupProbe: {} - # Fixed pod count, rendered into the Deployment's spec.replicas only when - # hpa.enabled is false. With the HPA on, the autoscaler owns the count and - # this value is ignored. - replicaCount: 1 + # Optional fixed pod count, rendered into the Deployment's spec.replicas only + # when hpa.enabled is false. Unset by default so an existing Deployment keeps + # its current count; with the HPA on, the autoscaler owns the count, e.g.: + # replicaCount: 3 + replicaCount: hpa: enabled: true minReplicas: 1 @@ -529,7 +530,7 @@ backend: # Optional startupProbe; same shape as gateway.startupProbe. Empty by default. startupProbe: {} # Same semantics as gateway.replicaCount. - replicaCount: 1 + replicaCount: hpa: enabled: true minReplicas: 1 @@ -597,7 +598,7 @@ ui: # Optional startupProbe; same shape as gateway.startupProbe. Empty by default. startupProbe: {} # Same semantics as gateway.replicaCount. - replicaCount: 1 + replicaCount: hpa: enabled: false minReplicas: 1 From 0317903a440117c0ffd4faba3639049147a2645e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 07:16:02 +0000 Subject: [PATCH 17/33] ci(e2e-changed): surface failed test ids from the pytest log Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-e2e-changed.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index f659afad7e3..6da16a33ea3 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -176,6 +176,7 @@ jobs: TESTS: ${{ needs.detect.outputs.tests }} E2E_FIXTURE_MODE: live E2E_PROVIDER_EDGE_HOST_REACHABLE: '1' + COLUMNS: '400' run: | umask 077 read -r -a test_files <<< "${TESTS}" @@ -190,6 +191,7 @@ jobs: uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}" verified=$? set -e + grep -E '^(FAILED|ERROR) ' "${log}" || true grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1 echo "::endgroup::" if [ "${status}" = "5" ]; then From b38504b5a6f646c7d4eaedb1847b32b4b18647b9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 07:37:28 +0000 Subject: [PATCH 18/33] test(e2e): assert the forwarded Converse body without requiring the model to accept safety_identifier Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../e2e/llm_translation/test_responses_e2e.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 1c2336bd166..525231de917 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -19,7 +19,6 @@ from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, un from e2e_http import ( assert_client_error, require_successful_call, - unwrap, ) from endpoints_client import ( EndpointsClient, @@ -359,20 +358,15 @@ class TestResponses: safety_identifier: Final = f"end-user-{unique_marker()}" if endpoint == "/v1/responses": - responses_result: Final = endpoints_client.responses( - key, model, "reply with one word", safety_identifier=safety_identifier - ) - require_successful_call(responses_result) + endpoints_client.responses(key, model, "reply with one word", safety_identifier=safety_identifier) else: - unwrap( - endpoints_client.proxy.chat( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content="reply with one word")], - safety_identifier=safety_identifier, - ), - ) + endpoints_client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="reply with one word")], + safety_identifier=safety_identifier, + ), ) forwarded: Final = tuple(body.additionalModelRequestFields for body in capture.bodies) From 8795be0a65754221bee0b318ad355843d5e3ca34 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 10:51:46 +0000 Subject: [PATCH 19/33] refactor(types): replace Any with proven types in 32 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/_logging.py | 8 +++++--- litellm/a2a_protocol/streaming_iterator.py | 12 ++++++------ litellm/integrations/focus/focus_logger.py | 4 +++- .../generic_prompt_manager.py | 2 +- .../mavvrik_focus/mavvrik_focus_logger.py | 17 ++++++++++++++--- litellm/integrations/otel/presets/agentops.py | 7 +++++-- litellm/integrations/otel/runtime.py | 13 ++++++++----- .../bounded_prometheus_series_tracker.py | 16 +++++++++++----- .../vector_store_pre_call_hook.py | 2 +- litellm/integrations/weights_biases.py | 9 +++++---- .../usage_object_transformation.py | 6 +++--- litellm/litellm_core_utils/safe_json_dumps.py | 2 +- litellm/llms/azure/fine_tuning/handler.py | 10 +++++----- .../llms/codestral/completion/transformation.py | 2 +- litellm/llms/databricks/common_utils.py | 4 ++-- litellm/llms/gemini/realtime/transformation.py | 4 ++-- .../llms/jina_ai/embedding/transformation.py | 2 +- .../llms/openrouter/embedding/transformation.py | 4 +++- litellm/llms/reducto/common.py | 4 +++- .../embedding/transformation.py | 3 ++- .../vertex_ai/agent_engine/transformation.py | 2 +- litellm/proxy/a2a/discovery.py | 7 ++++--- .../proxy/agent_endpoints/databricks_oauth.py | 11 ++++++----- litellm/proxy/client/cli/main.py | 13 +++++++++++-- .../generic_guardrail_api/__init__.py | 2 +- .../guardrails/guardrail_hooks/onyx/onyx.py | 4 ++-- litellm/proxy/hooks/responses_id_security.py | 6 +++--- .../callback_logs_endpoints.py | 5 +++-- .../management_v1/spend_logs.py | 8 ++++---- .../object_permission_utils.py | 4 ++-- .../gemini_passthrough_logging_handler.py | 2 +- .../proxy/public_endpoints/public_endpoints.py | 10 +++++++--- 32 files changed, 127 insertions(+), 78 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 5ba0c080364..644a79d8cbd 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -5,6 +5,7 @@ import logging import os import re import sys +from collections.abc import Sequence from datetime import datetime from logging import Formatter from typing import Any, Final, TextIO @@ -186,7 +187,8 @@ class SecretRedactionFilter(logging.Filter): record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place # Redact extra fields passed via logger.debug("msg", extra={...}) - for key, value in list(record.__dict__.items()): + record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items()) + for key, value in record_items: if key in _STANDARD_RECORD_ATTRS: continue if isinstance(value, str): @@ -507,7 +509,7 @@ handler.addFilter(_secret_filter) handler.addFilter(_correlation_filter) -def _try_parse_json_message(message: str) -> dict[str, Any] | None: +def _try_parse_json_message(message: str) -> dict[str, object] | None: """ Try to parse a log message as JSON. Returns parsed dict if valid, else None. Handles messages that are entirely valid JSON (e.g. json.dumps output). @@ -585,7 +587,7 @@ class JsonFormatter(Formatter): def format(self, record): message_str: Final = record.getMessage() - json_record: Final[dict[str, Any]] = { + json_record: Final[dict[str, object]] = { "message": message_str, "level": record.levelname, "timestamp": self.formatTime(record), diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index d936caeb75e..8232d7cf2d8 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -5,7 +5,7 @@ A2A Streaming Iterator with token tracking and logging support. import asyncio from collections.abc import AsyncIterator from datetime import datetime -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_logger @@ -15,7 +15,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj if TYPE_CHECKING: - from a2a.types import SendStreamingMessageRequest, SendStreamingMessageResponse + from a2a.compat.v0_3.types import SendStreamingMessageRequest, SendStreamingMessageResponse class A2AStreamingIterator: @@ -39,9 +39,9 @@ class A2AStreamingIterator: self.start_time = datetime.now() # Collect chunks for token counting - self.chunks: list[Any] = [] + self.chunks: list[SendStreamingMessageResponse] = [] self.collected_text_parts: list[str] = [] - self.final_chunk: Any | None = None + self.final_chunk: SendStreamingMessageResponse | None = None def __aiter__(self): return self @@ -69,7 +69,7 @@ class A2AStreamingIterator: await self._handle_stream_complete() raise - def _collect_text_from_chunk(self, chunk: Any) -> None: + def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse") -> None: """Extract text from a streaming chunk and add to collected parts.""" try: chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {} @@ -79,7 +79,7 @@ class A2AStreamingIterator: except Exception: verbose_logger.debug("Failed to extract text from A2A streaming chunk") - def _is_completed_chunk(self, chunk: Any) -> bool: + def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse") -> bool: """Check if chunk indicates stream completion.""" try: chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {} diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index c9b47835948..dce51b8190b 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -15,6 +15,8 @@ from .destinations import FocusTimeWindow if TYPE_CHECKING: from apscheduler.schedulers.asyncio import AsyncIOScheduler + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager + from .export_engine import FocusExportEngine else: AsyncIOScheduler = Any @@ -111,7 +113,7 @@ class FocusLogger(CustomLogger): """Entry point for scheduler jobs to run export cycle with locking.""" from litellm.proxy.proxy_server import proxy_logging_obj - pod_lock_manager = None + pod_lock_manager: PodLockManager | None = None if proxy_logging_obj is not None: writer: Final = getattr(proxy_logging_obj, "db_spend_update_writer", None) if writer is not None: diff --git a/litellm/integrations/generic_prompt_management/generic_prompt_manager.py b/litellm/integrations/generic_prompt_management/generic_prompt_manager.py index 77d315d0cee..de01b2bb02c 100644 --- a/litellm/integrations/generic_prompt_management/generic_prompt_manager.py +++ b/litellm/integrations/generic_prompt_management/generic_prompt_manager.py @@ -58,7 +58,7 @@ class GenericPromptManager(CustomPromptManagement): api_key: str | None = None, timeout: int = 30, prompt_id: str | None = None, - additional_provider_specific_query_params: dict[str, Any] | None = None, + additional_provider_specific_query_params: Mapping[str, object] | None = None, **kwargs, ): """ diff --git a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py index 3c189b4d53e..7e3c4cc3ce8 100644 --- a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py +++ b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py @@ -21,7 +21,7 @@ from __future__ import annotations import os from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol import litellm from litellm._logging import verbose_proxy_logger @@ -35,6 +35,17 @@ else: AsyncIOScheduler = Any +class _PodLockManager(Protocol): + """The subset of PodLockManager this logger drives to serialize the export across pods.""" + + @property + def redis_cache(self) -> object: ... + + async def acquire_lock(self, cronjob_id: str) -> bool | None: ... + + async def release_lock(self, cronjob_id: str) -> None: ... + + def _parse_metrics_marker( marker: object | None, ) -> datetime | None: @@ -226,9 +237,9 @@ class MavvrikFocusLogger(FocusLogger): """Scheduler entry point — uses Mavvrik-specific pod-lock key.""" from litellm.proxy.proxy_server import proxy_logging_obj # noqa: PLC0415 - pod_lock_manager = None + pod_lock_manager: _PodLockManager | None = None if proxy_logging_obj is not None: - writer: Final = getattr(proxy_logging_obj, "db_spend_update_writer", None) + writer: Final[object] = getattr(proxy_logging_obj, "db_spend_update_writer", None) if writer is not None: pod_lock_manager = getattr(writer, "pod_lock_manager", None) diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 965213f2ee4..58123656caa 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -9,9 +9,12 @@ this preset registers a custom exporter (``kind="agentops"``) that mints the JWT worker thread, off any event loop — and caches it for the process lifetime. """ +from collections.abc import Sequence from typing import Any, Final import httpx +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -71,7 +74,7 @@ def agentops_preset( ) -def _build_agentops_exporter(spec: ExporterSpec) -> Any: +def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter.""" from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, @@ -106,7 +109,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> Any: except Exception as e: verbose_logger.debug("AgentOps JWT fetch failed: %s", e) - def export(self, spans: Any) -> Any: + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: self._ensure_authenticated() return super().export(spans) diff --git a/litellm/integrations/otel/runtime.py b/litellm/integrations/otel/runtime.py index c6eaecd108b..13903597e1a 100644 --- a/litellm/integrations/otel/runtime.py +++ b/litellm/integrations/otel/runtime.py @@ -8,13 +8,16 @@ identity unconditionally. """ from collections.abc import Callable, Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager from functools import cache -from typing import Any, Final +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from opentelemetry.trace import Span @cache -def _otel_runtime() -> "tuple[Callable[[str], Any], Callable[..., None]] | None": +def _otel_runtime() -> "tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None]] | None": """Resolve the SDK-backed hooks once and cache the outcome, absence included. CPython never caches a failed import, so without this memoization every call @@ -29,7 +32,7 @@ def _otel_runtime() -> "tuple[Callable[[str], Any], Callable[..., None]] | None" @contextmanager -def phase_span(name: str) -> "Iterator[Any]": +def phase_span(name: str) -> "Iterator[Span | None]": """Run a request phase inside a live active span so its DB/service calls nest. Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not @@ -43,7 +46,7 @@ def phase_span(name: str) -> "Iterator[Any]": yield span -def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None: +def seed_request_identity(user_api_key_dict: object, model: object = None) -> None: """Seed request-identity Baggage at the auth boundary (no-op without V2).""" runtime: Final = _otel_runtime() if runtime is None: diff --git a/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py b/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py index c1ccf09d5d6..ba7d54fafea 100644 --- a/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py +++ b/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py @@ -3,7 +3,13 @@ from __future__ import annotations import time from collections import OrderedDict from threading import RLock -from typing import Any, Final +from typing import Final, Protocol + + +class _RemovableMetric(Protocol): + """The one prometheus-client metric method this tracker calls.""" + + def remove(self, *labelvalues: object) -> None: ... class BoundedPrometheusSeriesTracker: @@ -21,7 +27,7 @@ class BoundedPrometheusSeriesTracker: def track_series( self, - metric: Any, + metric: _RemovableMetric, metric_name: str, label_values: tuple[str | None, ...], max_series: int | None, @@ -60,7 +66,7 @@ class BoundedPrometheusSeriesTracker: break del series[tracked_label_values] - def remove_series(self, metric: object, label_values: tuple[str | None, ...]) -> bool: + def remove_series(self, metric: _RemovableMetric, label_values: tuple[str | None, ...]) -> bool: """Drop one child series, True when it is gone (removed or never existed).""" return self._remove_metric_child(metric, label_values) @@ -82,7 +88,7 @@ class BoundedPrometheusSeriesTracker: def _remove_metric_series( self, - metric: Any, + metric: _RemovableMetric, series: OrderedDict[tuple[str | None, ...], float], label_values: tuple[str | None, ...], ) -> None: @@ -90,7 +96,7 @@ class BoundedPrometheusSeriesTracker: series.pop(label_values, None) @staticmethod - def _remove_metric_child(metric: Any, label_values: tuple[str | None, ...]) -> bool: + def _remove_metric_child(metric: _RemovableMetric, label_values: tuple[str | None, ...]) -> bool: """ Remove the Prometheus child for ``label_values`` and report whether the tracker should commit the matching state change. diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index b2243060c6c..74fb8a8d6a3 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -406,7 +406,7 @@ class VectorStorePreCallHook(CustomLogger): request_data: dict, response_chunk: Any, call_type: CallTypes | None, - ) -> Any | None: + ) -> object | None: """ Add search results to the final streaming chunk. diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index d1a8ec098cf..9bc070a1f9a 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -4,6 +4,7 @@ imported_openAIResponse = True try: import io import logging + from collections.abc import Mapping from typing import Any, Literal, Protocol, TypeVar from wandb.sdk.data_types import trace_tree @@ -43,7 +44,7 @@ try: @staticmethod def results_to_trace_tree( - request: dict[str, Any], + request: Mapping[str, object], response: OpenAIResponse, results: list[trace_tree.Result], time_elapsed: float, @@ -73,7 +74,7 @@ try: def _resolve_edit( self, - request: dict[str, Any], + request: Mapping[str, object], response: OpenAIResponse, time_elapsed: float, ) -> trace_tree.WBTraceTree: @@ -91,7 +92,7 @@ try: def _resolve_completion( self, - request: dict[str, Any], + request: Mapping[str, object], response: OpenAIResponse, time_elapsed: float, ) -> trace_tree.WBTraceTree: @@ -134,7 +135,7 @@ try: def _request_response_result_to_trace( self, - request: dict[str, Any], + request: Mapping[str, object], response: OpenAIResponse, request_str: str, choices: list[str], diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index f11f6d46fb2..a02c40b7611 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -50,7 +50,7 @@ _INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType( ) -def _modality_field(entry: Mapping[str, Any]) -> str | None: +def _modality_field(entry: Mapping[str, object]) -> str | None: return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower()) @@ -58,7 +58,7 @@ def _token_count(value: object) -> int: return value if isinstance(value, int) else 0 -def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]: +def _modality_token_sums(entries: Sequence[Mapping[str, object]]) -> Mapping[str, int]: fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None) return MappingProxyType( { @@ -68,7 +68,7 @@ def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, i ) -def _google_search_query_count(usage_object: Mapping[str, Any]) -> int: +def _google_search_query_count(usage_object: Mapping[str, object]) -> int: entries: Final = usage_object.get("grounding_tool_count") if not isinstance(entries, Sequence): return 0 diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 4f9ac82d57d..63242a580e7 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -85,7 +85,7 @@ def safe_json_structure( def safe_dumps( - data: Any, + data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, ) -> str: diff --git a/litellm/llms/azure/fine_tuning/handler.py b/litellm/llms/azure/fine_tuning/handler.py index ac1e430e063..36c4fae04c7 100644 --- a/litellm/llms/azure/fine_tuning/handler.py +++ b/litellm/llms/azure/fine_tuning/handler.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine -from typing import Any, Final, cast +from typing import Final, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI @@ -19,7 +19,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): """ @staticmethod - def _ensure_training_type(create_fine_tuning_job_data: dict[str, Any]) -> None: + def _ensure_training_type(create_fine_tuning_job_data: dict[str, object]) -> None: """ Azure requires trainingType in extra_body. Default to 1 (supervised) if omitted. """ @@ -66,7 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, - ) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]: + ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: self._ensure_training_type(create_fine_tuning_job_data) openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( @@ -109,7 +109,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, - ) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]: + ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -149,7 +149,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, - ) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]: + ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, api_base=api_base, diff --git a/litellm/llms/codestral/completion/transformation.py b/litellm/llms/codestral/completion/transformation.py index e3c3fd1231c..baa134bb398 100644 --- a/litellm/llms/codestral/completion/transformation.py +++ b/litellm/llms/codestral/completion/transformation.py @@ -29,7 +29,7 @@ class CodestralTextCompletionConfig(OpenAITextCompletionConfig): random_seed: int | None = None, stop: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index a4ec2c5378b..f2f9df422e8 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -12,7 +12,7 @@ Authentication priority: import os import re -from typing import Any, Final, Literal +from typing import Final, Literal from urllib.parse import urlsplit, urlunsplit from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -48,7 +48,7 @@ class DatabricksBase: ] @classmethod - def redact_sensitive_data(cls, data: Any) -> Any: + def redact_sensitive_data(cls, data: object) -> object: """ Redact sensitive information (tokens, secrets) from data before logging. diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 79985569c5f..e64cbf88d95 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -453,7 +453,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return normalized @staticmethod - def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]: + def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]: generation_config: Final = setup.get("generationConfig") if isinstance(generation_config, dict): modalities: Final = generation_config.get("responseModalities") @@ -1172,7 +1172,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def map_openai_event( self, key: str, - value: Any, + value: object, current_delta_type: ALL_DELTA_TYPES | None, ) -> OpenAIRealtimeEventTypes | ResponsesAPIStreamEvents: if isinstance(value, dict): diff --git a/litellm/llms/jina_ai/embedding/transformation.py b/litellm/llms/jina_ai/embedding/transformation.py index 26f512979e5..260d9e6e494 100644 --- a/litellm/llms/jina_ai/embedding/transformation.py +++ b/litellm/llms/jina_ai/embedding/transformation.py @@ -31,7 +31,7 @@ class JinaAIEmbeddingConfig(BaseEmbeddingConfig): def __init__( self, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/openrouter/embedding/transformation.py b/litellm/llms/openrouter/embedding/transformation.py index 29d0c8c1c56..14b0e462ea7 100644 --- a/litellm/llms/openrouter/embedding/transformation.py +++ b/litellm/llms/openrouter/embedding/transformation.py @@ -170,7 +170,9 @@ class OpenrouterEmbeddingConfig(BaseEmbeddingConfig): optional_params[param] = value return optional_params - def get_error_class(self, error_message: str, status_code: int, headers: Any) -> Any: + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, str] | httpx.Headers + ) -> OpenRouterException: """ Get the error class for OpenRouter errors. """ diff --git a/litellm/llms/reducto/common.py b/litellm/llms/reducto/common.py index 9b9efd24b72..b194590fdb9 100644 --- a/litellm/llms/reducto/common.py +++ b/litellm/llms/reducto/common.py @@ -3,6 +3,8 @@ import binascii from collections import defaultdict from typing import TYPE_CHECKING, Any, Final, NoReturn +import httpx + from litellm.constants import request_timeout REDUCTO_API_BASE: Final = "https://platform.reducto.ai" @@ -62,7 +64,7 @@ def extract_file_id_or_bytes( return None, raw_bytes, mime -def _extract_file_id_from_upload_response(response: Any) -> str: +def _extract_file_id_from_upload_response(response: httpx.Response) -> str: try: payload: Final = response.json() except ValueError as exc: diff --git a/litellm/llms/vercel_ai_gateway/embedding/transformation.py b/litellm/llms/vercel_ai_gateway/embedding/transformation.py index 3f228c0881d..fc9c6bcc19f 100644 --- a/litellm/llms/vercel_ai_gateway/embedding/transformation.py +++ b/litellm/llms/vercel_ai_gateway/embedding/transformation.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllEmbeddingInputValues @@ -160,7 +161,7 @@ class VercelAIGatewayEmbeddingConfig(BaseEmbeddingConfig): optional_params[param] = value return optional_params - def get_error_class(self, error_message: str, status_code: int, headers: Any) -> Any: + def get_error_class(self, error_message: str, status_code: int, headers: Any) -> BaseLLMException: """ Get the error class for Vercel AI Gateway errors. """ diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index e430d9e2280..c5ca9f38144 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -205,7 +205,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): session_id: Final = self._get_session_id(optional_params) # Build the input - input_data: Final[dict[str, Any]] = { + input_data: Final[dict[str, str]] = { "message": prompt, "user_id": user_id, } diff --git a/litellm/proxy/a2a/discovery.py b/litellm/proxy/a2a/discovery.py index e08c938f195..ff58c9c85ec 100644 --- a/litellm/proxy/a2a/discovery.py +++ b/litellm/proxy/a2a/discovery.py @@ -14,6 +14,7 @@ fetcher dispatches by ``discovery_mode``: pure-A2A fallback strategy returns 404 for these deployments. """ +from collections.abc import Mapping from enum import Enum from typing import Any, Final from urllib.parse import urlencode @@ -55,7 +56,7 @@ def _normalize_base_url(base_url: str) -> str: def _build_langgraph_platform_paths( - params: dict[str, Any] | None, + params: Mapping[str, object] | None, ) -> tuple[str, ...]: """Build the paths to try for LangGraph Platform discovery. @@ -71,7 +72,7 @@ def _build_langgraph_platform_paths( return tuple(f"{path}?{query}" for path in AGENT_CARD_WELL_KNOWN_PATHS) -def _paths_for_mode(mode: DiscoveryMode, params: dict[str, Any] | None) -> tuple[str, ...]: +def _paths_for_mode(mode: DiscoveryMode, params: Mapping[str, object] | None) -> tuple[str, ...]: if mode == DiscoveryMode.WELL_KNOWN_FALLBACK: return AGENT_CARD_WELL_KNOWN_PATHS if mode == DiscoveryMode.LANGGRAPH_PLATFORM: @@ -83,7 +84,7 @@ async def fetch_well_known_card( base_url: str, *, discovery_mode: DiscoveryMode = DiscoveryMode.WELL_KNOWN_FALLBACK, - params: dict[str, Any] | None = None, + params: Mapping[str, object] | None = None, timeout: float = DEFAULT_DISCOVERY_TIMEOUT_SECONDS, headers: dict[str, str] | None = None, ) -> dict[str, Any]: diff --git a/litellm/proxy/agent_endpoints/databricks_oauth.py b/litellm/proxy/agent_endpoints/databricks_oauth.py index 4c3b1bc084d..38a76ea6890 100644 --- a/litellm/proxy/agent_endpoints/databricks_oauth.py +++ b/litellm/proxy/agent_endpoints/databricks_oauth.py @@ -25,8 +25,9 @@ Config example:: import asyncio import base64 import hashlib +from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Final +from typing import Final import httpx @@ -43,7 +44,7 @@ _TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60 _DEFAULT_TTL_SECONDS: Final = 3600 -def _resolve_secret(value: Any) -> str | None: +def _resolve_secret(value: object) -> str | None: """Resolve a config value, expanding ``os.environ/`` references.""" if not isinstance(value, str): return None @@ -75,7 +76,7 @@ class DatabricksAppOAuthConfig: def parse_databricks_oauth_config( - litellm_params: dict[str, Any] | None, + litellm_params: Mapping[str, object] | None, ) -> DatabricksAppOAuthConfig | None: """Build a Databricks App OAuth config from an agent's ``litellm_params``. @@ -191,7 +192,7 @@ class DatabricksAppOAuthTokenCache(InMemoryCache): except httpx.HTTPError as exc: raise ValueError(f"Databricks App OAuth token request failed: {exc}") from exc - body: Final = response.json() + body: Final[object] = response.json() if not isinstance(body, dict): raise ValueError( f"Databricks App OAuth token response returned non-object JSON (got {type(body).__name__})" @@ -215,7 +216,7 @@ databricks_app_oauth_token_cache: Final = DatabricksAppOAuthTokenCache() async def resolve_databricks_app_auth_header( - litellm_params: dict[str, Any] | None, + litellm_params: Mapping[str, object] | None, ) -> dict[str, str] | None: """Return ``{"Authorization": "Bearer "}`` for a Databricks App agent. diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 63e38c93221..6d63acc7479 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -9,7 +9,15 @@ from litellm._version import version as litellm_version from litellm.proxy.client.health import HealthManagementClient from .commands.agents import agent_commands -from .commands.auth import auth_group, context_secret_vault, get_stored_api_key, login, logout, whoami +from .commands.auth import ( + CliContextObj, + auth_group, + context_secret_vault, + get_stored_api_key, + login, + logout, + whoami, +) from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names @@ -126,7 +134,8 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s @click.pass_context def version(ctx: click.Context): """Show the LiteLLM Proxy CLI and server version.""" - print_version(ctx.obj.get("base_url"), ctx.obj.get("api_key")) + ctx_obj: Final[CliContextObj] = ctx.obj + print_version(ctx_obj.get("base_url"), ctx_obj.get("api_key")) # Add authentication commands as top-level commands diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py index d1576b68813..e3511d46544 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams -def _get_config_value(litellm_params: Any, optional_params: Any, attribute_name: str) -> Any | None: +def _get_config_value(litellm_params: "LitellmParams", optional_params: object, attribute_name: str) -> Any | None: if optional_params is not None: value: Final = ( optional_params.get(attribute_name) diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 7529c4ce3f3..1a6feb47215 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -6,7 +6,7 @@ # +-------------------------------------------------------------+ import os import uuid -from typing import TYPE_CHECKING, Any, Final, Literal, Optional +from typing import TYPE_CHECKING, Final, Literal, Optional import httpx from fastapi import HTTPException @@ -63,7 +63,7 @@ class OnyxGuardrail(CustomGuardrail): async def _validate_with_guard_server( self, - payload: Any, + payload: object, input_type: Literal["request", "response"], conversation_id: str, ) -> dict: diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index d9050489095..bdf7e2ab53d 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -40,7 +40,7 @@ _UNMANAGED_RESPONSE_ID_DETAIL: Final = ( _PROXY_ADMIN_ROLES: Final = frozenset({LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN.value}) -def _proxy_general_settings() -> Mapping[str, Any]: +def _proxy_general_settings() -> Mapping[str, object]: from litellm.proxy.proxy_server import general_settings return general_settings @@ -107,7 +107,7 @@ def _is_responses_api_create_route(request_route: str | None) -> bool: class ResponsesIDSecurity(CustomLogger): def __init__( self, - general_settings_reader: Callable[[], Mapping[str, Any]] = _proxy_general_settings, + general_settings_reader: Callable[[], Mapping[str, object]] = _proxy_general_settings, signing_key_reader: Callable[[], str | None] = _proxy_signing_key, ) -> None: self._general_settings_reader: Final = general_settings_reader @@ -307,7 +307,7 @@ class ResponsesIDSecurity(CustomLogger): data: dict, user_api_key_dict: "UserAPIKeyAuth", response: LLMResponseTypes, - ) -> Any: + ) -> LLMResponseTypes: """ Queue response IDs for batch processing instead of writing directly to DB. diff --git a/litellm/proxy/logging_endpoints/callback_logs_endpoints.py b/litellm/proxy/logging_endpoints/callback_logs_endpoints.py index cecadc03d71..4a1079871b0 100644 --- a/litellm/proxy/logging_endpoints/callback_logs_endpoints.py +++ b/litellm/proxy/logging_endpoints/callback_logs_endpoints.py @@ -15,6 +15,7 @@ self-describing `StandardLoggingPayload`, so completions/responses can use it to """ import uuid +from collections.abc import Mapping from datetime import datetime, timezone from typing import Any, Final @@ -48,7 +49,7 @@ class CallbackLogsReplayer: """ @staticmethod - def _epoch_to_datetime(value: Any) -> datetime: + def _epoch_to_datetime(value: object) -> datetime: """`StandardLoggingPayload` stores startTime/endTime as float epoch seconds.""" if isinstance(value, (int, float)): return datetime.fromtimestamp(float(value), tz=timezone.utc) @@ -114,7 +115,7 @@ class CallbackLogsReplayer: return logging_obj @staticmethod - def _response_obj_from_payload(payload: dict[str, Any]) -> dict[str, Any]: + def _response_obj_from_payload(payload: Mapping[str, object]) -> dict[str, object]: """Minimal response object so usage-derived spend-log fields resolve.""" return { "id": payload.get("id"), diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index f6907a7f87a..1cbc454ca5e 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,7 +1,7 @@ """`/management/v1/spend_logs` facets.""" from datetime import datetime, timezone -from typing import Annotated, Any, Final, Literal +from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Query, Request @@ -39,7 +39,7 @@ async def _spend_log_scope_clause( user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, next_param_index: int, -) -> tuple[str | None, tuple[Any, ...]]: +) -> tuple[str | None, tuple[str | list[str], ...]]: """SQL predicate restricting the facet to spend logs this caller may read. Returns ``(None, ())`` for a proxy admin. Mirrors the scoping ``/spend/logs/ui`` @@ -101,8 +101,8 @@ async def _list_spend_log_facet( ) column_sql: Final = "end_user" if column == "end_user" else '"user"' - window_params: Final[tuple[Any, ...]] = (_as_utc(start_time), _as_utc(end_time)) - search_params: Final[tuple[Any, ...]] = (f"%{escape_like(q)}%",) if q else () + window_params: Final[tuple[datetime, datetime]] = (_as_utc(start_time), _as_utc(end_time)) + search_params: Final[tuple[str, ...]] = (f"%{escape_like(q)}%",) if q else () search_clause: Final = (f"{column_sql} ILIKE ${len(window_params) + 1} ESCAPE '\\'",) if q else () scope_clause, scope_params = await _spend_log_scope_clause( diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index daab38d3662..437e6763502 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -8,7 +8,7 @@ from collections.abc import Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Final, Optional from fastapi import HTTPException, status from pydantic import TypeAdapter @@ -230,7 +230,7 @@ def _dedupe_preserving_order(values: list[str]) -> list[str]: return result -def _mcp_server_identifier_matches(server: Any, identifier: str) -> bool: +def _mcp_server_identifier_matches(server: object, identifier: str) -> bool: return identifier in { getattr(server, "server_id", None), getattr(server, "alias", None), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index a95ee87fd31..d97ddb9a909 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -147,7 +147,7 @@ class GeminiPassthroughLoggingHandler: - Creates standard logging object - Logs in litellm callbacks """ - kwargs: dict[str, Any] = {} + kwargs: dict[str, object] = {} model = model or GeminiPassthroughLoggingHandler.extract_model_from_url(url_route) complete_streaming_response: Final = GeminiPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index e395f56194f..26a5c44fce1 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -199,9 +199,13 @@ def _build_endpoints(raw: _ProvidersFile) -> list[_EndpointEntry]: return result +_PROVIDERS_FILE_ADAPTER: Final = TypeAdapter(_ProvidersFile) +_PROVIDER_CREATE_FIELDS_ADAPTER: Final = TypeAdapter(list[ProviderCreateInfo]) + + def _load_endpoints() -> list[_EndpointEntry]: - raw: Final[_ProvidersFile] = json.loads( - files("litellm").joinpath("provider_endpoints_support_backup.json").read_text(encoding="utf-8") + raw: Final = _PROVIDERS_FILE_ADAPTER.validate_python( + json.loads(files("litellm").joinpath("provider_endpoints_support_backup.json").read_text(encoding="utf-8")) ) return _build_endpoints(raw) @@ -398,7 +402,7 @@ async def get_provider_fields() -> list[ProviderCreateInfo]: ) with open(provider_create_fields_path, "r") as f: - provider_create_fields: Final = json.load(f) + provider_create_fields: Final = _PROVIDER_CREATE_FIELDS_ADAPTER.validate_python(json.load(f)) return provider_create_fields From 85e72a2e2fac84e9be99cbac4ede9d4991b834af Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:00:50 +0000 Subject: [PATCH 20/33] chore(prices): sync OpenRouter prices: 2 models openrouter/deepseek/deepseek-v4-flash: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost --- litellm/model_prices_and_context_window_backup.json | 12 ++++++------ model_prices_and_context_window.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1976437f900..d3053dee25d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 9.53172e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.906344e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.9431e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 5.852e-08, + "output_cost_per_token": 1.1704e-07, + "cache_read_input_token_cost": 1.1704e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1976437f900..d3053dee25d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 9.53172e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.906344e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.9431e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 5.852e-08, + "output_cost_per_token": 1.1704e-07, + "cache_read_input_token_cost": 1.1704e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, From d9a97d74db83d791276e8b8099412a1d87e930bf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:21:50 +0000 Subject: [PATCH 21/33] fix(model_prices): add groq qwen3.6-27b deprecation date and bedrock qwen3-next regional pricing Groq lists qwen/qwen3.6-27b for shutdown on 2026-09-14. Adds the six regional Bedrock qwen.qwen3-next-80b-a3b entries priced per AWS's published regional rates (absorbs #42191) with a regression test that the regional entry is used instead of the US rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 79 +++++++++++++++++++ model_prices_and_context_window.json | 79 +++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 27 +++++++ 3 files changed, 185 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 176df53f706..856396a1d75 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -35294,6 +35294,7 @@ "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "deprecation_date": "2026-09-14", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -44299,6 +44300,84 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.45e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.41e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.545e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.236e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.41e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.45e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, "litellm_provider": "bedrock_converse", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 176df53f706..856396a1d75 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -35294,6 +35294,7 @@ "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "deprecation_date": "2026-09-14", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -44299,6 +44300,84 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.45e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.41e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.545e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.236e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.41e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, + "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.45e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true + }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, "litellm_provider": "bedrock_converse", diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index aef17f3d5d0..da3d022d669 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4230,3 +4230,30 @@ def test_completion_cost_prices_responses_websocket_turns_per_service_tier(): assert ws_cost == pytest.approx(_http_cost(100, 40, "default") + _http_cost(60, 10, "priority")) assert ws_cost != pytest.approx(_http_cost(160, 50, "default")) assert ws_cost != pytest.approx(_http_cost(160, 50, "priority")) + + +QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") + + +@pytest.mark.parametrize("region", QWEN3_NEXT_REGIONS) +def test_cost_per_token_bedrock_qwen3_next_uses_regional_entry_not_us_rate( + monkeypatch: pytest.MonkeyPatch, region: str +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + regional: Final = litellm.model_cost[f"bedrock/{region}/qwen.qwen3-next-80b-a3b"] + us: Final = litellm.model_cost["qwen.qwen3-next-80b-a3b"] + assert regional["input_cost_per_token"] != us["input_cost_per_token"] + assert regional["output_cost_per_token"] != us["output_cost_per_token"] + + prompt_tokens, completion_tokens = 1000, 500 + prompt_usd, completion_usd = cost_per_token( + model=f"bedrock/{region}/qwen.qwen3-next-80b-a3b", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + custom_llm_provider="bedrock", + ) + + assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) + assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) From 2888b4f5f4d74e246a11d9fda9f01cea9a6309b2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:30:35 +0000 Subject: [PATCH 22/33] fix(bedrock): whitelist regional qwen3-next keys for converse routing check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/local_testing/whitelisted_bedrock_models.txt | 6 ++++++ whitelisted_bedrock_models.txt | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt index 762d655b886..7615a540b23 100644 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ b/tests/local_testing/whitelisted_bedrock_models.txt @@ -133,3 +133,9 @@ meta.llama3-2-11b-instruct-v1:0 us.meta.llama3-2-11b-instruct-v1:0 meta.llama3-2-90b-instruct-v1:0 us.meta.llama3-2-90b-instruct-v1:0 +bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b +bedrock/ap-south-1/qwen.qwen3-next-80b-a3b +bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b +bedrock/eu-west-1/qwen.qwen3-next-80b-a3b +bedrock/eu-west-2/qwen.qwen3-next-80b-a3b +bedrock/sa-east-1/qwen.qwen3-next-80b-a3b diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 6124cb41044..254842e2714 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -44,6 +44,7 @@ bedrock/ap-northeast-1/minimax.minimax-m2.5 bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking bedrock/ap-northeast-1/moonshotai.kimi-k2.5 bedrock/ap-northeast-1/qwen.qwen3-coder-next +bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b bedrock/moonshotai.kimi-k2-thinking bedrock/moonshotai.kimi-k2.5 bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0 @@ -54,6 +55,7 @@ bedrock/ap-south-1/minimax.minimax-m2.5 bedrock/ap-south-1/moonshotai.kimi-k2-thinking bedrock/ap-south-1/moonshotai.kimi-k2.5 bedrock/ap-south-1/qwen.qwen3-coder-next +bedrock/ap-south-1/qwen.qwen3-next-80b-a3b bedrock/ap-southeast-2/minimax.minimax-m2.5 bedrock/ap-southeast-3/deepseek.v3.2 bedrock/ap-southeast-3/minimax.minimax-m2.1 @@ -83,11 +85,13 @@ bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0 bedrock/eu-west-1/minimax.minimax-m2.1 bedrock/eu-west-1/minimax.minimax-m2.5 bedrock/eu-west-1/qwen.qwen3-coder-next +bedrock/eu-west-1/qwen.qwen3-next-80b-a3b bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0 bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0 bedrock/eu-west-2/minimax.minimax-m2.1 bedrock/eu-west-2/minimax.minimax-m2.5 bedrock/eu-west-2/qwen.qwen3-coder-next +bedrock/eu-west-2/qwen.qwen3-next-80b-a3b bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2 bedrock/eu-west-3/mistral.mistral-large-2402-v1:0 bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1 @@ -103,6 +107,7 @@ bedrock/sa-east-1/minimax.minimax-m2.5 bedrock/sa-east-1/moonshotai.kimi-k2-thinking bedrock/sa-east-1/moonshotai.kimi-k2.5 bedrock/sa-east-1/qwen.qwen3-coder-next +bedrock/sa-east-1/qwen.qwen3-next-80b-a3b bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1 bedrock/us-east-1/1-month-commitment/anthropic.claude-v1 bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1 @@ -240,3 +245,4 @@ bedrock/us-gov-east-1/anthropic.claude-sonnet-5 bedrock/us-gov-east-1/anthropic.claude-opus-4-8 bedrock/us-gov-east-1/anthropic.claude-opus-5 bedrock/us-gov-east-1/anthropic.claude-fable-5-1 +bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b From 6db2bce43ce28f6cb8d32540347b576c379c7e84 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:30:42 +0000 Subject: [PATCH 23/33] chore(prices): sync OpenRouter prices: 1 model openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d3053dee25d..d3f05f77793 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.53172e-07, + "input_cost_per_token": 9.51432e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.906344e-06, + "output_cost_per_token": 1.902864e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9431e-08, + "cache_read_input_token_cost": 7.9286e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d3053dee25d..d3f05f77793 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.53172e-07, + "input_cost_per_token": 9.51432e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.906344e-06, + "output_cost_per_token": 1.902864e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9431e-08, + "cache_read_input_token_cost": 7.9286e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, From 6786eb0131c5da5aabd8372726be27c70f0a1712 Mon Sep 17 00:00:00 2001 From: yassin Date: Mon, 21 Sep 2026 13:39:32 +0000 Subject: [PATCH 24/33] fix(a2a): send message/stream for Bedrock AgentCore streaming requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/bedrock_agentcore/handler.py | 2 +- .../test_bedrock_agentcore_a2a.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py index 306a8871b12..da5eb522187 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -98,7 +98,7 @@ class BedrockAgentCoreA2AHandler: request_id=request_id, params=params, litellm_params=litellm_params, - method="message/send", + method="message/stream", stream=True, agent_extra_headers=agent_extra_headers, ) diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py index a8fe464ec32..256d73e3612 100644 --- a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py +++ b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -605,6 +605,38 @@ class TestNonStreaming: assert result["error"]["message"] == "Bad request" +class TestStreaming: + """Streaming requests must ask AgentCore for a stream, not a single send.""" + + @pytest.mark.asyncio + async def test_streaming_request_uses_message_stream_method_and_yields_sse_events(self, httpx_transport): + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + sse_body = ( + 'data: {"jsonrpc": "2.0", "id": "req-001", "result": {"kind": "task", "id": "t1"}}\n\n' + 'data: {"jsonrpc": "2.0", "id": "req-001", "result": {"kind": "status-update", "final": true}}\n\n' + ) + with respx.mock(assert_all_called=True) as router: + route = router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=sse_body) + ) + events = [ + event + async for event in BedrockAgentCoreA2AConfig().handle_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + ] + + sent_body = json.loads(route.calls.last.request.content) + assert sent_body["method"] == "message/stream", sent_body + assert sent_body["params"]["message"]["messageId"] == "msg-001" + assert [event["result"]["kind"] for event in events] == ["task", "status-update"] + + class TestConfigManager: """Test that config manager routes 'bedrock' correctly.""" From 18ca95c3a7973008f308ba044f3ff4d34402f238 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:00:51 +0000 Subject: [PATCH 25/33] chore(prices): sync OpenRouter prices: 1 model openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d3f05f77793..39ef6510954 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.51432e-07, + "input_cost_per_token": 9.48126e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.902864e-06, + "output_cost_per_token": 1.896252e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9286e-08, + "cache_read_input_token_cost": 7.90105e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d3f05f77793..39ef6510954 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.51432e-07, + "input_cost_per_token": 9.48126e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.902864e-06, + "output_cost_per_token": 1.896252e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9286e-08, + "cache_read_input_token_cost": 7.90105e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, From 346ad002c8bc0178e04e4adee654e2a2adc4fdc7 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:30:57 +0000 Subject: [PATCH 26/33] chore(prices): sync OpenRouter prices: 4 models openrouter/~deepseek/deepseek-pro-latest: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing openrouter/deepseek/deepseek-v4-flash: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro-0813: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing openrouter/meta-llama/llama-3.1-70b-instruct: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token --- ...odel_prices_and_context_window_backup.json | 38 +++++++++---------- model_prices_and_context_window.json | 38 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 39ef6510954..2d22376e2ea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41579,22 +41579,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 1.32e-06, + "input_cost_per_token": 5.7948e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 3.96e-06, + "output_cost_per_token": 1.73844e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 4.4e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, + "cache_read_input_token_cost": 1.8438e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.852e-08, - "output_cost_per_token": 1.1704e-07, - "cache_read_input_token_cost": 1.1704e-08, + "input_cost_per_token": 5.698e-08, + "output_cost_per_token": 1.1396e-07, + "cache_read_input_token_cost": 1.1396e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69010,12 +69010,12 @@ "supports_reasoning": false }, "openrouter/meta-llama/llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -71317,15 +71317,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 4.4e-08, - "input_cost_per_token": 1.32e-06, + "cache_read_input_token_cost": 1.8438e-08, + "input_cost_per_token": 5.7948e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, - "output_cost_per_token": 3.96e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, + "output_cost_per_token": 1.73844e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 39ef6510954..2d22376e2ea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41579,22 +41579,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 1.32e-06, + "input_cost_per_token": 5.7948e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 3.96e-06, + "output_cost_per_token": 1.73844e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 4.4e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, + "cache_read_input_token_cost": 1.8438e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.852e-08, - "output_cost_per_token": 1.1704e-07, - "cache_read_input_token_cost": 1.1704e-08, + "input_cost_per_token": 5.698e-08, + "output_cost_per_token": 1.1396e-07, + "cache_read_input_token_cost": 1.1396e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69010,12 +69010,12 @@ "supports_reasoning": false }, "openrouter/meta-llama/llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "output_cost_per_token": 7.2e-07, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -71317,15 +71317,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 4.4e-08, - "input_cost_per_token": 1.32e-06, + "cache_read_input_token_cost": 1.8438e-08, + "input_cost_per_token": 5.7948e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, - "output_cost_per_token": 3.96e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, + "output_cost_per_token": 1.73844e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, From 7b2d3b36b6c7fb205e2b24b0e49220a337905cc8 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 21 Sep 2026 14:40:46 +0000 Subject: [PATCH 27/33] fix(prices): align deepseek-v4-pro-0813 cache hit cost with cache read cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2d22376e2ea..7a5c28c9df4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41580,7 +41580,7 @@ }, "openrouter/deepseek/deepseek-v4-pro-0813": { "input_cost_per_token": 5.7948e-07, - "input_cost_per_token_cache_hit": 4.4e-08, + "input_cost_per_token_cache_hit": 1.8438e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2d22376e2ea..7a5c28c9df4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41580,7 +41580,7 @@ }, "openrouter/deepseek/deepseek-v4-pro-0813": { "input_cost_per_token": 5.7948e-07, - "input_cost_per_token_cache_hit": 4.4e-08, + "input_cost_per_token_cache_hit": 1.8438e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, From 1ac4d7ae042129f29aaf1a0d05b20b823888f24e Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 21 Sep 2026 09:55:09 -0500 Subject: [PATCH 28/33] fix(anthropic): type safeguards and safeguard_results as the arrays Anthropic sends Driving a real Claude Code 2.1.278 through the proxy, and a direct call to api.anthropic.com, both show these two fields are JSON arrays on the wire rather than objects. The request carries safeguards as [{"type": "dangerous_tool_use", "classifier_context": {...}}] under beta dangerous-tool-use-2026-09-03, and the 200 comes back with safeguard_results as [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {...}}}]. No runtime change: the request filter matches on TypedDict keys and never inspects the value. The test fixtures move to the captured shapes so the regression tests pin what the client and the provider actually exchange. --- litellm/types/llms/anthropic.py | 6 +++--- .../anthropic_messages/anthropic_response.py | 2 +- .../test_handler_output_config_passthrough.py | 2 +- ...experimental_pass_through_messages_handler.py | 16 ++++++++++------ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f57591d0262..c59c88698f7 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -411,7 +411,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): output_config: AnthropicOutputConfig | None # Configuration for Claude's output behavior cache_control: dict[str, Any] | None # Automatic prompt caching reasoning_effort: str | None - safeguards: ReadOnly[dict[str, object] | None] + safeguards: ReadOnly[list[dict[str, object]] | None] class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): @@ -531,7 +531,7 @@ class AnthropicStopDetails(TypedDict, total=False): class MessageDelta(TypedDict, total=False): stop_reason: str | None stop_details: ReadOnly[AnthropicStopDetails] - safeguard_results: ReadOnly[dict[str, object]] + safeguard_results: ReadOnly[list[dict[str, object]]] class ServerToolUsage(TypedDict, total=False): @@ -602,7 +602,7 @@ class MessageChunk(TypedDict, total=False): stop_reason: str | None stop_sequence: str | None usage: UsageDelta - safeguard_results: ReadOnly[dict[str, object]] + safeguard_results: ReadOnly[list[dict[str, object]]] class MessageStartBlock(TypedDict): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 41060e96d85..1d4c3cdc864 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -97,4 +97,4 @@ class AnthropicMessagesResponse(TypedDict, total=False): type: Literal["message"] | None usage: AnthropicUsage | None context_management: NotRequired[ContextManagementResponse] - safeguard_results: NotRequired[ReadOnly[dict[str, object]]] + safeguard_results: NotRequired[ReadOnly[list[dict[str, object]]]] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py index d6de6372e0b..6246f502344 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py @@ -113,7 +113,7 @@ class TestOutputConfigStrippedFromCompletionKwargs: def test_safeguards_is_stripped_for_non_anthropic_target(self): extra_kwargs = { "custom_llm_provider": "azure", - "safeguards": {"auto_mode": {"enabled": True, "version": "2026-09-01"}}, + "safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}], } result = _call_prepare(extra_kwargs=extra_kwargs) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 0acb9d634a3..e8bfcb86bf6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1442,10 +1442,12 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped(): @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): + """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" from litellm.llms.anthropic.experimental_pass_through.messages import handler - safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} - client_betas = "safeguards-2026-09-01,interleaved-thinking-2025-05-14" + safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] + client_betas = "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14" + safeguard_results = [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {}}}] captured: dict[str, object] = {} def upstream_records_the_request(request: httpx.Request) -> httpx.Response: @@ -1462,7 +1464,7 @@ async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthro "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}, - "safeguard_results": {"verdict": "allow"}, + "safeguard_results": safeguard_results, }, request=request, ) @@ -1483,15 +1485,17 @@ async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthro assert captured["body"]["safeguards"] == safeguards assert set(captured["anthropic-beta"].split(",")) == set(client_betas.split(",")) - assert response["safeguard_results"] == {"verdict": "allow"} + assert response["safeguard_results"] == safeguard_results @pytest.mark.asyncio async def test_anthropic_messages_streaming_forwards_safeguards_and_keeps_safeguard_results(): + """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" from litellm.llms.anthropic.experimental_pass_through.messages import handler - safeguards = {"auto_mode": {"enabled": True, "version": "2026-09-01"}} - safeguard_results = {"verdict": "allow", "checks": ["shell_command"]} + safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] + tool_verdicts = {"toolu_01": {"type": "evaluated", "outcome": "not_flagged"}} + safeguard_results = [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": tool_verdicts}}] captured: dict[str, object] = {} message_start = { "type": "message_start", From 33e64e53f9d1590b0ea48b45459a6232f098ba65 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:00:53 +0000 Subject: [PATCH 29/33] chore(prices): sync OpenRouter prices: 4 models openrouter/~deepseek/deepseek-pro-latest: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing openrouter/deepseek/deepseek-v4-flash: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro-0813: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing --- ...odel_prices_and_context_window_backup.json | 36 +++++++++---------- model_prices_and_context_window.json | 36 +++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7a5c28c9df4..ef581d447eb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.48126e-07, + "input_cost_per_token": 9.46386e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.896252e-06, + "output_cost_per_token": 1.892772e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.90105e-08, + "cache_read_input_token_cost": 7.88655e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41579,22 +41579,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.7948e-07, + "input_cost_per_token": 5.7816e-07, "input_cost_per_token_cache_hit": 1.8438e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.73844e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.8438e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, + "cache_read_input_token_cost": 1.9272e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.698e-08, - "output_cost_per_token": 1.1396e-07, - "cache_read_input_token_cost": 1.1396e-08, + "input_cost_per_token": 5.544e-08, + "output_cost_per_token": 1.1088e-07, + "cache_read_input_token_cost": 1.1088e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -71317,15 +71317,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.8438e-08, - "input_cost_per_token": 5.7948e-07, + "cache_read_input_token_cost": 1.9272e-08, + "input_cost_per_token": 5.7816e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, - "output_cost_per_token": 1.73844e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7a5c28c9df4..ef581d447eb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41537,21 +41537,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.48126e-07, + "input_cost_per_token": 9.46386e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.896252e-06, + "output_cost_per_token": 1.892772e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.90105e-08, + "cache_read_input_token_cost": 7.88655e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41579,22 +41579,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.7948e-07, + "input_cost_per_token": 5.7816e-07, "input_cost_per_token_cache_hit": 1.8438e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.73844e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.8438e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, + "cache_read_input_token_cost": 1.9272e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -67259,9 +67259,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.698e-08, - "output_cost_per_token": 1.1396e-07, - "cache_read_input_token_cost": 1.1396e-08, + "input_cost_per_token": 5.544e-08, + "output_cost_per_token": 1.1088e-07, + "cache_read_input_token_cost": 1.1088e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -71317,15 +71317,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.8438e-08, - "input_cost_per_token": 5.7948e-07, + "cache_read_input_token_cost": 1.9272e-08, + "input_cost_per_token": 5.7816e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7948e-7,"output_cost_per_token":0.00000173844,"cache_read_input_token_cost":1.8438e-8}, - "output_cost_per_token": 1.73844e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, From 97fc8220dfd083725f540dda4230702a8bc483b8 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 21 Sep 2026 15:14:19 +0000 Subject: [PATCH 30/33] fix(prices): align deepseek-v4-pro-0813 off-peak and cache-hit rates with the base cache read rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef581d447eb..1f569ad1fbc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41580,7 +41580,7 @@ }, "openrouter/deepseek/deepseek-v4-pro-0813": { "input_cost_per_token": 5.7816e-07, - "input_cost_per_token_cache_hit": 1.8438e-08, + "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -41594,7 +41594,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.9272e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -71324,7 +71324,7 @@ "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef581d447eb..1f569ad1fbc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41580,7 +41580,7 @@ }, "openrouter/deepseek/deepseek-v4-pro-0813": { "input_cost_per_token": 5.7816e-07, - "input_cost_per_token_cache_hit": 1.8438e-08, + "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -41594,7 +41594,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.9272e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -71324,7 +71324,7 @@ "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.8438e-8}, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, From a3dcec463b9e64d424496644b1b9cdc57dbd049c Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:31:08 +0000 Subject: [PATCH 31/33] chore(prices): sync Azure prices: 1 model, 1 deprecated azure_ai/MAI-Image-2.5-Pro: deprecation_date --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3bc8b3c176c..91e869cbb36 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11175,6 +11175,7 @@ "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Pro": { + "deprecation_date": "2026-10-01", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3bc8b3c176c..91e869cbb36 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11175,6 +11175,7 @@ "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Pro": { + "deprecation_date": "2026-10-01", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", From ade39978a9202d28ad53088207a6143f16c4c0ad Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:31:16 +0000 Subject: [PATCH 32/33] chore(prices): sync AWS Bedrock prices: 25 models [enrichment failed: AWS Bedrock, 66 held] anthropic.claude-fable-5: anthropic.claude-fable-5-1: anthropic.claude-opus-4-7: anthropic.claude-opus-4-8: anthropic.claude-opus-5: anthropic.claude-sonnet-4-6: anthropic.claude-sonnet-5: global.anthropic.claude-fable-5: global.anthropic.claude-fable-5-1: global.anthropic.claude-opus-4-7: global.anthropic.claude-opus-4-8: global.anthropic.claude-opus-5: global.anthropic.claude-sonnet-4-6: global.anthropic.claude-sonnet-5: us-gov.anthropic.claude-fable-5-1: us-gov.anthropic.claude-opus-4-8: us-gov.anthropic.claude-opus-5: us-gov.anthropic.claude-sonnet-5: us.anthropic.claude-fable-5: us.anthropic.claude-fable-5-1: us.anthropic.claude-opus-4-7: us.anthropic.claude-opus-4-8: us.anthropic.claude-opus-5: us.anthropic.claude-sonnet-4-6: us.anthropic.claude-sonnet-5: --- ...odel_prices_and_context_window_backup.json | 50 +++++++++---------- model_prices_and_context_window.json | 50 +++++++++---------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3bc8b3c176c..98e570d7874 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1327,7 +1327,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1381,7 +1381,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1419,7 +1419,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1531,7 +1531,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1570,7 +1570,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1608,7 +1608,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1647,7 +1647,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1685,7 +1685,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1724,7 +1724,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1837,7 +1837,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1875,7 +1875,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1913,7 +1913,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2063,7 +2063,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2102,7 +2102,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2141,7 +2141,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2329,7 +2329,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2368,7 +2368,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2407,7 +2407,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2556,7 +2556,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2591,7 +2591,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2626,7 +2626,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -47289,7 +47289,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47323,7 +47323,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47356,7 +47356,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47407,7 +47407,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3bc8b3c176c..98e570d7874 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1327,7 +1327,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1381,7 +1381,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1419,7 +1419,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1531,7 +1531,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1570,7 +1570,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1608,7 +1608,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1647,7 +1647,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1685,7 +1685,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1724,7 +1724,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1837,7 +1837,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1875,7 +1875,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1913,7 +1913,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2063,7 +2063,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2102,7 +2102,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2141,7 +2141,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2329,7 +2329,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2368,7 +2368,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2407,7 +2407,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2556,7 +2556,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2591,7 +2591,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2626,7 +2626,7 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -47289,7 +47289,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47323,7 +47323,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47356,7 +47356,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -47407,7 +47407,7 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, From 739227fefc8eb4824c59988f131c92599b6fdd30 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:31:20 +0000 Subject: [PATCH 33/33] chore(prices): sync OpenRouter prices: 3 models openrouter/~deepseek/deepseek-pro-latest: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro-0813: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing --- ...odel_prices_and_context_window_backup.json | 30 +++++++++---------- model_prices_and_context_window.json | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3bc8b3c176c..4d92825b34f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41590,21 +41590,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.46386e-07, + "input_cost_per_token": 9.42906e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.892772e-06, + "output_cost_per_token": 1.885812e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.88655e-08, + "cache_read_input_token_cost": 7.85755e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41632,22 +41632,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.7816e-07, + "input_cost_per_token": 5.7684e-07, "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.73448e-06, + "output_cost_per_token": 1.73052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.9272e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, + "cache_read_input_token_cost": 1.8354e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7684e-7,"output_cost_per_token":0.00000173052,"cache_read_input_token_cost":1.8354e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -71484,15 +71484,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.9272e-08, - "input_cost_per_token": 5.7816e-07, + "cache_read_input_token_cost": 1.8354e-08, + "input_cost_per_token": 5.7684e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, - "output_cost_per_token": 1.73448e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7684e-7,"output_cost_per_token":0.00000173052,"cache_read_input_token_cost":1.8354e-8}, + "output_cost_per_token": 1.73052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3bc8b3c176c..4d92825b34f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41590,21 +41590,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.46386e-07, + "input_cost_per_token": 9.42906e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.892772e-06, + "output_cost_per_token": 1.885812e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.88655e-08, + "cache_read_input_token_cost": 7.85755e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41632,22 +41632,22 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.7816e-07, + "input_cost_per_token": 5.7684e-07, "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.73448e-06, + "output_cost_per_token": 1.73052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.9272e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, + "cache_read_input_token_cost": 1.8354e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7684e-7,"output_cost_per_token":0.00000173052,"cache_read_input_token_cost":1.8354e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -71484,15 +71484,15 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.9272e-08, - "input_cost_per_token": 5.7816e-07, + "cache_read_input_token_cost": 1.8354e-08, + "input_cost_per_token": 5.7684e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7816e-7,"output_cost_per_token":0.00000173448,"cache_read_input_token_cost":1.9272e-8}, - "output_cost_per_token": 1.73448e-06, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.7684e-7,"output_cost_per_token":0.00000173052,"cache_read_input_token_cost":1.8354e-8}, + "output_cost_per_token": 1.73052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true,