From bfc7ae05bf78982b7cce01e5af37ad4b6e8574b5 Mon Sep 17 00:00:00 2001 From: unknown <> Date: Wed, 1 Jul 2026 14:20:26 +0000 Subject: [PATCH 01/81] feat: add claude-sonnet-5 pricing to model cost map Add Claude Sonnet 5 with introductory pricing (/0 per MTok, through Aug 31 2026) across Anthropic, Bedrock (global + regional), Vertex AI, Azure AI, and Snowflake providers. Includes cache pricing (1.25x write, 2x 1hr write, 0.1x read) and 128K output context. Fixes #31868 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + ...odel_prices_and_context_window_backup.json | 1836 +++++++++++------ model_prices_and_context_window.json | 1398 +++++++------ .../test_claude_sonnet_5_config.py | 163 ++ 4 files changed, 2137 insertions(+), 1261 deletions(-) create mode 100644 tests/test_litellm/test_claude_sonnet_5_config.py diff --git a/litellm/constants.py b/litellm/constants.py index aeb74a65839..1d17e7b6da6 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1128,6 +1128,7 @@ BEDROCK_CONVERSE_MODELS = [ "anthropic.claude-opus-4-6-v1:0", "anthropic.claude-opus-4-6-v1", "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", "anthropic.claude-opus-4-1-20250805-v1:0", "anthropic.claude-opus-4-20250514-v1:0", "anthropic.claude-sonnet-4-20250514-v1:0", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 21132db93cb..15b16118a7b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1010,7 +1010,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1042,7 +1041,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1074,7 +1072,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1106,7 +1103,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1138,7 +1134,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1170,7 +1165,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1219,7 +1213,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1253,7 +1246,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1287,7 +1279,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1321,7 +1312,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1487,7 +1477,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1521,7 +1510,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1555,7 +1543,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1589,7 +1576,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1623,7 +1609,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1688,7 +1673,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1719,7 +1703,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1750,7 +1733,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1781,7 +1763,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1812,7 +1793,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1843,7 +1823,186 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "anthropic.claude-sonnet-5": { "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "global.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "us.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "eu.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "au.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "jp.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2364,7 +2523,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2394,7 +2552,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2455,7 +2612,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2523,6 +2679,34 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "azure_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -9132,17 +9316,16 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9754,17 +9937,16 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -10274,6 +10456,35 @@ "supports_vision": true, "supports_output_config": true }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -16134,6 +16345,46 @@ "tpm": 8000000, "supports_image_size": false }, + "gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -16174,6 +16425,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-image": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, @@ -17588,6 +17877,48 @@ }, "supports_image_size": false }, + "gemini/gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "rpm": 1000, + "tpm": 4000000, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -17630,6 +17961,47 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-image": { + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "tpm": 4000000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3.1-flash-image-preview": { "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -18944,7 +19316,6 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_adaptive_thinking": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true @@ -21696,8 +22067,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21745,8 +22116,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21790,8 +22161,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21835,8 +22206,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21884,8 +22255,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21932,8 +22303,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21973,8 +22344,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22017,8 +22388,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22054,7 +22425,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -22062,8 +22433,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22100,7 +22471,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -22108,8 +22479,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22144,15 +22515,15 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22187,15 +22558,15 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22227,9 +22598,9 @@ "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 272000, - "max_tokens": 272000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -22263,9 +22634,9 @@ "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 272000, - "max_tokens": 272000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -24712,14 +25083,13 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-06, "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -28175,7 +28545,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28215,7 +28584,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28277,7 +28645,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -29522,6 +29889,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/z-ai/glm-5.1": { + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 5.25e-07, + "cache_creation_input_token_cost": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.1", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/minimax/minimax-m2.1": { "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.2e-06, @@ -30200,7 +30583,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -30210,7 +30592,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -31126,7 +31507,7 @@ "supports_tool_choice": true }, "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", + "deprecation_date": "2025-06-25", "input_cost_per_token": 4e-08, "litellm_provider": "sambanova", "max_input_tokens": 16384, @@ -31257,15 +31638,15 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, "supports_computer_use": true, "supports_function_calling": true, "supports_vision": true, @@ -31273,14 +31654,14 @@ "supports_system_messages": true, "supports_response_schema": true }, - "snowflake/deepseek-r1": { + "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "supports_reasoning": true, "supports_system_messages": true }, @@ -31339,8 +31720,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.0000012, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 1.2e-06, "supports_function_calling": true, "supports_system_messages": true }, @@ -31350,8 +31731,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true }, @@ -31361,8 +31742,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000024, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, "supports_system_messages": true }, "snowflake/llama3.2-1b": { @@ -31379,17 +31760,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/llama3.3-70b": { + "snowflake/llama3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, @@ -31404,14 +31785,14 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/mistral-large2": { + "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_response_schema": true @@ -31451,17 +31832,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/snowflake-llama-3.3-70b": { + "snowflake/snowflake-llama-3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", @@ -32713,7 +33094,7 @@ "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, - "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", @@ -33455,7 +33836,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34690,7 +35070,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34720,7 +35099,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34750,7 +35128,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34781,7 +35158,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34872,7 +35248,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34903,7 +35278,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34956,7 +35330,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -35265,6 +35638,21 @@ "tpm": 8000000, "supports_image_size": false }, + "vertex_ai/gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" + }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -35280,6 +35668,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, @@ -37485,12 +37886,12 @@ }, "zai.glm-5": { "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.2e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -37511,20 +37912,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -37540,6 +37927,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-5.1": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-5-code": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-07, @@ -37570,6 +37972,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-4.7-flash": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 0, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.6": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -42393,7 +42810,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -42411,6 +42827,64 @@ }, "supports_output_config": true }, + "vertex_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "vertex_ai/claude-sonnet-5@default": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "duckduckgo/search": { "litellm_provider": "duckduckgo", "mode": "search", @@ -42427,7 +42901,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42442,7 +42919,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42457,7 +42937,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42471,7 +42953,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42487,9 +42971,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42507,9 +42998,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42526,7 +43024,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42542,7 +43043,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42558,7 +43062,10 @@ "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42717,20 +43224,6 @@ } ] }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -42759,45 +43252,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-east-1/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-west-2/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, @@ -42844,362 +43298,382 @@ "supports_native_structured_output": true, "supports_pdf_input": true }, - "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-sonnet-4-6": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-opus": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 0.0000001, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.000000125, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, - "max_input_tokens": 1000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000012, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, - "max_input_tokens": 5000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000006, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/llama4-maverick": { - "max_tokens": 16384, - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000097, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, - "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "soniox/stt-async-v4": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "soniox/stt-async-v5": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 6e-07, - "output_cost_per_token": 3.6e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 1.8e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3.6-27B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 3.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 202752, - "max_output_tokens": 202752, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/moonshotai/Kimi-K2.6": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 9.6e-07, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/google/gemma-4-31B-it": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 5.6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-120b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-20b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - } - , + "snowflake/claude-sonnet-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-4-6": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-5": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-opus": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/claude-haiku-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-3-7-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-4.1": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-mini": { + "max_tokens": 16384, + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-nano": { + "max_tokens": 16384, + "max_input_tokens": 5000000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/llama4-maverick": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, + "snowflake/snowflake-arctic-embed-l-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "snowflake/snowflake-arctic-embed-m-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3.6-27B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/moonshotai/Kimi-K2.6": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 9.6e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/google/gemma-4-31B-it": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-120b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-20b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-09, @@ -43275,6 +43749,109 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-pro": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 3.625e-09, + "input_cost_per_token": 4.35e-07, + "input_cost_per_token_cache_hit": 3.625e-09, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "pinstripes/ps/glm-4.5-air": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3.6-35b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3-30b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3-coder-30b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": false, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/deepseek-v4-flash": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/minimax-m2.7": { + "max_tokens": 1000192, + "max_input_tokens": 1000192, + "max_output_tokens": 1000192, + "input_cost_per_token": 2.55e-07, + "output_cost_per_token": 5.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": false, + "source": "https://pinstripes.io/pricing" + }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, "litellm_provider": "darkbloom", @@ -43309,142 +43886,39 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "deepseek/deepseek-v4-pro": { - "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, - "litellm_provider": "deepseek", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.7e-07, - "source": "https://api-docs.deepseek.com/quick_start/pricing", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.00000045, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000045, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.00000009, - "output_cost_per_token": 0.0000002, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, - "max_input_tokens": 1000192, - "max_output_tokens": 1000192, - "input_cost_per_token": 0.000000255, - "output_cost_per_token": 0.00000055, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" - }, - "fallback_generalizations": { - "rules": [ - { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", - "model_info": { - "supports_adaptive_thinking": true - } - }, - { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", - "model_info": { - "litellm_provider": "anthropic", - "mode": "chat", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_tool_choice": true, - "supports_assistant_prefill": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_reasoning": true, - "supports_pdf_input": true, - "supports_system_messages": true - } - } - ] - } + "fallback_generalizations": { + "rules": [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_tool_choice": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_pdf_input": true, + "supports_system_messages": true + } + } + ] + } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 73cefeb7c77..15b16118a7b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1010,7 +1010,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1042,7 +1041,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1074,7 +1072,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1106,7 +1103,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1138,7 +1134,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1170,7 +1165,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1219,7 +1213,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1253,7 +1246,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1287,7 +1279,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1321,7 +1312,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1487,7 +1477,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1521,7 +1510,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1555,7 +1543,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1589,7 +1576,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1623,7 +1609,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1688,7 +1673,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1719,7 +1703,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1750,7 +1733,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1781,7 +1763,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1812,7 +1793,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1843,7 +1823,186 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "anthropic.claude-sonnet-5": { "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "global.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "us.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "eu.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "au.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "jp.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2364,7 +2523,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2394,7 +2552,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2455,7 +2612,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2523,6 +2679,34 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "azure_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -9132,17 +9316,16 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9754,17 +9937,16 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -10274,6 +10456,35 @@ "supports_vision": true, "supports_output_config": true }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -19105,7 +19316,6 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_adaptive_thinking": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true @@ -21857,8 +22067,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21906,8 +22116,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21951,8 +22161,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21996,8 +22206,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22045,8 +22255,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22093,8 +22303,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22134,8 +22344,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22178,8 +22388,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22223,8 +22433,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22269,8 +22479,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22312,8 +22522,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22355,8 +22565,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -24873,14 +25083,13 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-06, "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -28336,7 +28545,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28376,7 +28584,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28438,7 +28645,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -30377,7 +30583,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -30387,7 +30592,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -31434,15 +31638,15 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, "supports_computer_use": true, "supports_function_calling": true, "supports_vision": true, @@ -31450,14 +31654,14 @@ "supports_system_messages": true, "supports_response_schema": true }, - "snowflake/deepseek-r1": { + "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "supports_reasoning": true, "supports_system_messages": true }, @@ -31516,8 +31720,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.0000012, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 1.2e-06, "supports_function_calling": true, "supports_system_messages": true }, @@ -31527,8 +31731,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true }, @@ -31538,8 +31742,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000024, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, "supports_system_messages": true }, "snowflake/llama3.2-1b": { @@ -31556,17 +31760,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/llama3.3-70b": { + "snowflake/llama3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, @@ -31581,14 +31785,14 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/mistral-large2": { + "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_response_schema": true @@ -31628,17 +31832,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/snowflake-llama-3.3-70b": { + "snowflake/snowflake-llama-3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", @@ -32890,7 +33094,7 @@ "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, - "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", @@ -33632,7 +33836,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34867,7 +35070,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34897,7 +35099,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34927,7 +35128,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34958,7 +35158,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35049,7 +35248,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35080,7 +35278,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35133,7 +35330,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -37690,12 +37886,12 @@ }, "zai.glm-5": { "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.2e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -37716,20 +37912,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -42628,7 +42810,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -42646,6 +42827,64 @@ }, "supports_output_config": true }, + "vertex_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "vertex_ai/claude-sonnet-5@default": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "duckduckgo/search": { "litellm_provider": "duckduckgo", "mode": "search", @@ -42662,7 +42901,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42677,7 +42919,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42692,7 +42937,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42706,7 +42953,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42722,9 +42971,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42742,9 +42998,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42761,7 +43024,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42777,7 +43043,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42793,7 +43062,10 @@ "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42952,20 +43224,6 @@ } ] }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -42994,45 +43252,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-east-1/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-west-2/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, @@ -43079,363 +43298,382 @@ "supports_native_structured_output": true, "supports_pdf_input": true }, - "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-sonnet-4-6": { - "supports_adaptive_thinking": true, - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-opus": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 0.0000001, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.000000125, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, - "max_input_tokens": 1000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000012, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, - "max_input_tokens": 5000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000006, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/llama4-maverick": { - "max_tokens": 16384, - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000097, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, - "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "soniox/stt-async-v4": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "soniox/stt-async-v5": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 6e-07, - "output_cost_per_token": 3.6e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 1.8e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3.6-27B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 3.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 202752, - "max_output_tokens": 202752, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/moonshotai/Kimi-K2.6": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 9.6e-07, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/google/gemma-4-31B-it": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 5.6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-120b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-20b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - } - , + "snowflake/claude-sonnet-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-4-6": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-5": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-opus": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/claude-haiku-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-3-7-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-4.1": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-mini": { + "max_tokens": 16384, + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-nano": { + "max_tokens": 16384, + "max_input_tokens": 5000000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/llama4-maverick": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, + "snowflake/snowflake-arctic-embed-l-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "snowflake/snowflake-arctic-embed-m-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3.6-27B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/moonshotai/Kimi-K2.6": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 9.6e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/google/gemma-4-31B-it": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-120b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-20b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-09, @@ -43540,8 +43778,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43553,8 +43791,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43566,8 +43804,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000009, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43579,8 +43817,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43592,8 +43830,8 @@ "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43605,8 +43843,8 @@ "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, - "input_cost_per_token": 0.000000255, - "output_cost_per_token": 0.00000055, + "input_cost_per_token": 2.55e-07, + "output_cost_per_token": 5.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43648,39 +43886,39 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "fallback_generalizations": { - "rules": [ - { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", - "model_info": { - "supports_adaptive_thinking": true - } - }, - { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", - "model_info": { - "litellm_provider": "anthropic", - "mode": "chat", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_tool_choice": true, - "supports_assistant_prefill": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_reasoning": true, - "supports_pdf_input": true, - "supports_system_messages": true - } - } - ] - } + "fallback_generalizations": { + "rules": [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_tool_choice": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_pdf_input": true, + "supports_system_messages": true + } + } + ] + } } diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py new file mode 100644 index 00000000000..943c4172205 --- /dev/null +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -0,0 +1,163 @@ +""" +Validate Claude Sonnet 5 model configuration entries. + +Sonnet 5 launched 2026-06-29 with introductory pricing ($2/$10 per MTok) +through August 31, 2026, after which it moves to standard pricing ($3/$15). +The cost-map entries use the introductory rates so spend tracking is accurate +during the promotional period. +""" + +import json +import os + +import pytest + +import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + +REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") + + +def _load_root_cost_map() -> dict: + json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") + with open(json_path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +def test_sonnet_5_model_pricing_and_capabilities(): + model_data = _load_root_cost_map() + + expected_models = [ + ("claude-sonnet-5", "anthropic"), + ("anthropic.claude-sonnet-5", "bedrock_converse"), + ("vertex_ai/claude-sonnet-5", "vertex_ai-anthropic_models"), + ("azure_ai/claude-sonnet-5", "azure_ai"), + ] + + for model_name, provider in expected_models: + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + + assert info["litellm_provider"] == provider + assert info["mode"] == "chat" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + assert info["max_tokens"] == 128000 + + assert info["input_cost_per_token"] == 2e-06 + assert info["output_cost_per_token"] == 1e-05 + assert info["cache_creation_input_token_cost"] == 2.5e-06 + assert info["cache_creation_input_token_cost_above_1hr"] == 4e-06 + assert info["cache_read_input_token_cost"] == 2e-07 + + assert "input_cost_per_token_above_200k_tokens" not in info + assert "output_cost_per_token_above_200k_tokens" not in info + + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + + +def test_sonnet_5_bedrock_regional_model_pricing(): + model_data = _load_root_cost_map() + + expected_models = { + "global.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_creation_input_token_cost": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + }, + "us.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "eu.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "au.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "jp.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + } + + for model_name, expected in expected_models.items(): + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + assert info["litellm_provider"] == "bedrock_converse" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + for key, value in expected.items(): + assert info[key] == value, f"{model_name}.{key}: expected {value}, got {info[key]}" + + +def test_sonnet_5_present_in_bundled_backup(): + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + for model_name in ( + "claude-sonnet-5", + "anthropic.claude-sonnet-5", + "global.anthropic.claude-sonnet-5", + "us.anthropic.claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + "au.anthropic.claude-sonnet-5", + "jp.anthropic.claude-sonnet-5", + "vertex_ai/claude-sonnet-5", + "vertex_ai/claude-sonnet-5@default", + "azure_ai/claude-sonnet-5", + "snowflake/claude-sonnet-5", + ): + assert model_name in backup, f"Missing from backup cost map: {model_name}" + assert backup[model_name] == root[model_name], model_name + + +def test_sonnet_5_provider_resolves_via_model_info(local_model_cost_map): + info = litellm.get_model_info(model="claude-sonnet-5") + assert info["litellm_provider"] == "anthropic" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + assert info["input_cost_per_token"] == 2e-06 + assert info["output_cost_per_token"] == 1e-05 + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map): + variants = [k for k in cost_map if "claude-sonnet-5" in k and "sonnet-4-5" not in k] + assert variants, "no claude-sonnet-5 entries found in cost map" + missing = [ + k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True + ] + assert not missing, f"missing supports_adaptive_thinking: {missing}" From 901a097b86ddb9539982173bcc91a6a29a27a2d7 Mon Sep 17 00:00:00 2001 From: unknown <> Date: Wed, 1 Jul 2026 14:35:36 +0000 Subject: [PATCH 02/81] fix: restore supports_reasoning on minimax entries lost during dedup The original JSON had duplicate minimax.minimax-m2.5 entries; the first had supports_reasoning: true, the second did not. JSON deduplication kept the second. Restore the flag on the surviving entries and add 1hr cache TTL assertions to the regional bedrock test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 ++++++--- model_prices_and_context_window.json | 9 ++++++--- tests/test_litellm/test_claude_sonnet_5_config.py | 5 +++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 15b16118a7b..dba8bb949d0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9325,7 +9325,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9946,7 +9947,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -25092,7 +25094,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15b16118a7b..dba8bb949d0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9325,7 +9325,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9946,7 +9947,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -25092,7 +25094,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 943c4172205..c8726a039f2 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -82,30 +82,35 @@ def test_sonnet_5_bedrock_regional_model_pricing(): "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, }, "us.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "eu.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "au.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "jp.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, } From bbf51146ca7ffa63e5a46c903fa13b14b7f48c59 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 13:29:38 +0000 Subject: [PATCH 03/81] fix(pricing): correct Vertex Haiku 4.5 output limit, Bedrock Mantle gpt-oss rates, add Scaleway deprecation dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 32 +++++++++++-------- model_prices_and_context_window.json | 32 +++++++++++-------- .../test_bedrock_mantle_transformation.py | 4 +-- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2459ed940e0..1bc4c275161 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45421,8 +45421,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45446,8 +45446,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51727,7 +51727,8 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/openai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, @@ -51766,7 +51767,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-01" }, "scaleway/hcompany/holo2-30b-a3b": { "input_cost_per_token": 3e-07, @@ -51777,7 +51779,8 @@ "mode": "chat", "output_cost_per_token": 7e-07, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-09" }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, @@ -51800,7 +51803,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/voxtral-small-24b-2507": { "input_cost_per_audio_token": 1.5e-07, @@ -51811,7 +51815,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.5e-07, - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, @@ -51833,7 +51838,8 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/BAAI/bge-multilingual-gemma2": { "input_cost_per_token": 1e-07, @@ -54283,7 +54289,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { - "input_cost_per_token": 7.5e-08, + "input_cost_per_token": 7e-08, "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, @@ -54317,8 +54323,8 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 65536, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2459ed940e0..1bc4c275161 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45421,8 +45421,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45446,8 +45446,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51727,7 +51727,8 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/openai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, @@ -51766,7 +51767,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-01" }, "scaleway/hcompany/holo2-30b-a3b": { "input_cost_per_token": 3e-07, @@ -51777,7 +51779,8 @@ "mode": "chat", "output_cost_per_token": 7e-07, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-09" }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, @@ -51800,7 +51803,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/voxtral-small-24b-2507": { "input_cost_per_audio_token": 1.5e-07, @@ -51811,7 +51815,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.5e-07, - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, @@ -51833,7 +51838,8 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/BAAI/bge-multilingual-gemma2": { "input_cost_per_token": 1e-07, @@ -54283,7 +54289,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { - "input_cost_per_token": 7.5e-08, + "input_cost_per_token": 7e-08, "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, @@ -54317,8 +54323,8 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 65536, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index b88c27e64b9..92c012d339c 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -696,8 +696,8 @@ class TestBedrockMantlePricing: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") litellm.add_known_models() info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-20b") - # Bedrock pricing: $0.075/M input, $0.30/M output - assert info["input_cost_per_token"] == pytest.approx(7.5e-8) + # Bedrock pricing: $0.07/M input, $0.30/M output + assert info["input_cost_per_token"] == pytest.approx(7e-8) assert info["output_cost_per_token"] == pytest.approx(3e-7) def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch): From 528f3ae0161192004ca76d615363c748dcfdcd0d Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 13:46:46 +0000 Subject: [PATCH 04/81] test(bedrock_mantle): cover gpt-oss-safeguard-20b input and output rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock_mantle/test_bedrock_mantle_transformation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 92c012d339c..b3eec97058a 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -700,6 +700,14 @@ class TestBedrockMantlePricing: assert info["input_cost_per_token"] == pytest.approx(7e-8) assert info["output_cost_per_token"] == pytest.approx(3e-7) + def test_gpt_oss_safeguard_20b_pricing(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-safeguard-20b") + # Bedrock pricing: $0.07/M input, $0.20/M output + assert info["input_cost_per_token"] == pytest.approx(7e-8) + assert info["output_cost_per_token"] == pytest.approx(2e-7) + def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch): """ Verify Bedrock Mantle pricing is cheaper than OpenAI's direct API pricing. From b4e7776ab3534b4c6b91073aea1a1bbf8f681e83 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 19:21:18 +0000 Subject: [PATCH 05/81] fix(model_prices): add gemini lyria-3.5, voyage-multilingual-2, chatgpt gpt-5.5 and gpt-5.6 entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 99 +++++++++++++++++++ model_prices_and_context_window.json | 99 +++++++++++++++++++ .../test_chatgpt_responses_transformation.py | 53 +++++++++- 3 files changed, 249 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1bc4c275161..8780c41b70a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27541,6 +27541,70 @@ "max_tokens": 8191, "mode": "embedding" }, + "chatgpt/gpt-5.5": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.5", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-luna": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-sol": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-terra": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", "max_input_tokens": 1050000, @@ -47654,6 +47718,16 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-multilingual-2": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-3-large": { "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", @@ -60044,6 +60118,31 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, + "gemini/lyria-3.5": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1bc4c275161..8780c41b70a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27541,6 +27541,70 @@ "max_tokens": 8191, "mode": "embedding" }, + "chatgpt/gpt-5.5": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.5", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-luna": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-sol": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-terra": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", "max_input_tokens": 1050000, @@ -47654,6 +47718,16 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-multilingual-2": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-3-large": { "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", @@ -60044,6 +60118,31 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, + "gemini/lyria-3.5": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 8e0415d50de..8a0027fdb53 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -10,18 +10,23 @@ from unittest.mock import MagicMock, patch import httpx import pytest - +import litellm +from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig from litellm.llms.openai.common_utils import OpenAIError +from litellm.main import responses_api_bridge_check from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager -from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig class TestChatGPTResponsesAPITransformation: @pytest.mark.parametrize( "model_name", [ + "chatgpt/gpt-5.5", + "chatgpt/gpt-5.6-luna", + "chatgpt/gpt-5.6-sol", + "chatgpt/gpt-5.6-terra", "chatgpt/gpt-5.4", "chatgpt/gpt-5.4-pro", "chatgpt/gpt-5.3-chat-latest", @@ -40,6 +45,50 @@ class TestChatGPTResponsesAPITransformation: assert isinstance(config, ChatGPTResponsesAPIConfig) assert config.custom_llm_provider == LlmProviders.CHATGPT + @pytest.mark.parametrize( + "model_name", + [ + "chatgpt/gpt-5.5", + "chatgpt/gpt-5.6-luna", + "chatgpt/gpt-5.6-sol", + "chatgpt/gpt-5.6-terra", + ], + ) + def test_chatgpt_responses_model_metadata(self, model_name, local_model_cost_map): + model_info = litellm.get_model_info(model_name) + + assert model_info["litellm_provider"] == "chatgpt" + assert model_info["mode"] == "responses" + assert model_info["supported_endpoints"] == [ + "/v1/chat/completions", + "/v1/responses", + ] + assert model_info["max_input_tokens"] == 1050000 + assert model_info["max_output_tokens"] == 128000 + + @pytest.mark.parametrize( + "model_name", + [ + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + ], + ) + def test_chatgpt_models_bridge_chat_completions_to_responses(self, model_name, local_model_cost_map): + """A chat completions request for these models must take the Responses bridge. + + `gpt-5.6-*` also exists as an openai chat model, so an unregistered + chatgpt model resolves to mode "chat" here and never reaches the bridge. + """ + model_info, resolved_model = responses_api_bridge_check( + model=model_name, + custom_llm_provider="chatgpt", + ) + + assert model_info["mode"] == "responses" + assert resolved_model == model_name + @patch("litellm.llms.chatgpt.responses.transformation.Authenticator") def test_chatgpt_responses_endpoint_url(self, mock_authenticator_class): mock_auth_instance = MagicMock() From 15c9fca53fc04c5ab05ebba0a8f047ef77394abd Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 13:17:30 +0000 Subject: [PATCH 06/81] fix(pricing): add xai grok-imagine-video entries, computer-use-preview deprecation, gemini live 2.5 native audio card values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 94 +++++++++++++++++-- model_prices_and_context_window.json | 94 +++++++++++++++++-- 2 files changed, 174 insertions(+), 14 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c558a953032..33e9ce94455 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14336,6 +14336,7 @@ "supports_tool_choice": true }, "computer-use-preview": { + "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -23829,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23854,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { @@ -59479,6 +59480,85 @@ "image" ] }, + "xai/grok-imagine-video": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.x.ai/docs/models/grok-imagine-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-2026-05-30": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, "low/1024-x-1024/grok-imagine-image-2.0": { "input_cost_per_image": 0.04, "litellm_provider": "xai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c558a953032..33e9ce94455 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14336,6 +14336,7 @@ "supports_tool_choice": true }, "computer-use-preview": { + "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -23829,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23854,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { @@ -59479,6 +59480,85 @@ "image" ] }, + "xai/grok-imagine-video": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.x.ai/docs/models/grok-imagine-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-2026-05-30": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, "low/1024-x-1024/grok-imagine-image-2.0": { "input_cost_per_image": 0.04, "litellm_provider": "xai", From 68d7a2d98fadd8f15eec9d5b0005e79de33e98bb Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 13:29:29 +0000 Subject: [PATCH 07/81] fix(pricing): drop supported_endpoints from xai grok-imagine-video entries to match video model convention Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ------------ model_prices_and_context_window.json | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 33e9ce94455..537db678f60 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59487,9 +59487,6 @@ "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, "source": "https://docs.x.ai/docs/models/grok-imagine-video", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59507,9 +59504,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59527,9 +59521,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59547,9 +59538,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 33e9ce94455..537db678f60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59487,9 +59487,6 @@ "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, "source": "https://docs.x.ai/docs/models/grok-imagine-video", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59507,9 +59504,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59527,9 +59521,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59547,9 +59538,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", From cb8daee55f05e3d4ec728c1b3f15e153176721ea Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 13:23:25 +0000 Subject: [PATCH 08/81] chore(registry): absorb #40185 daybreak-blue and #40159 cloudflare rpm limits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 93 ++++++++++++++++--- model_prices_and_context_window.json | 93 ++++++++++++++++--- tests/test_litellm/test_utils.py | 6 ++ 3 files changed, 164 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5de376218b9..ef8da798922 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13792,7 +13792,8 @@ "max_output_tokens": 3072, "max_tokens": 3072, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/meta/llama-2-7b-chat-int8": { "input_cost_per_token": 1.923e-06, @@ -13801,7 +13802,8 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { "input_cost_per_token": 1.923e-06, @@ -13810,7 +13812,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { "input_cost_per_token": 1.923e-06, @@ -13819,7 +13822,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/openai/gpt-oss-120b": { "input_cost_per_token": 3.5e-07, @@ -13829,6 +13833,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13839,7 +13844,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-3b-instruct": { "input_cost_per_token": 5.09e-08, @@ -13848,7 +13854,8 @@ "max_output_tokens": 80000, "max_tokens": 80000, "mode": "chat", - "output_cost_per_token": 3.35e-07 + "output_cost_per_token": 3.35e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-guard-3-8b": { "input_cost_per_token": 4.84e-07, @@ -13857,7 +13864,8 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-08 + "output_cost_per_token": 3e-08, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { "input_cost_per_token": 0.0, @@ -13866,7 +13874,8 @@ "max_output_tokens": 15000, "max_tokens": 15000, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -13877,6 +13886,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13888,6 +13898,7 @@ "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, + "rpm": 300, "supports_reasoning": true }, "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { @@ -13897,7 +13908,8 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 2.87e-07 + "output_cost_per_token": 2.87e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-1b-instruct": { "input_cost_per_token": 2.7e-08, @@ -13906,7 +13918,8 @@ "max_output_tokens": 60000, "max_tokens": 60000, "mode": "chat", - "output_cost_per_token": 2.01e-07 + "output_cost_per_token": 2.01e-07, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.6": { "cache_read_input_token_cost": 1.6e-07, @@ -13917,6 +13930,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13928,6 +13942,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13938,7 +13953,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { "input_cost_per_token": 2.93e-07, @@ -13948,6 +13964,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { @@ -13958,6 +13975,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { @@ -13967,7 +13985,8 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "rpm": 300 }, "cloudflare/@cf/zai-org/glm-5.2": { "cache_read_input_token_cost": 2.6e-07, @@ -13978,6 +13997,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13989,6 +14009,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13999,7 +14020,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.55e-07 + "output_cost_per_token": 5.55e-07, + "rpm": 300 }, "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { "input_cost_per_token": 5.09e-08, @@ -14009,6 +14031,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14019,7 +14042,8 @@ "max_output_tokens": 3500, "max_tokens": 3500, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1e-07, @@ -14029,6 +14053,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14040,6 +14065,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { @@ -14050,6 +14076,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, + "rpm": 300, "supports_vision": true }, "cloudflare/@cf/openai/gpt-oss-20b": { @@ -14060,6 +14087,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14071,6 +14099,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwq-32b": { @@ -14081,6 +14110,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, + "rpm": 300, "supports_reasoning": true }, "codestral/codestral-2405": { @@ -54901,6 +54931,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html" + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -60822,6 +60885,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -60832,6 +60896,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", "supported_endpoints": [ "/v1/audio/transcriptions" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5de376218b9..ef8da798922 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13792,7 +13792,8 @@ "max_output_tokens": 3072, "max_tokens": 3072, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/meta/llama-2-7b-chat-int8": { "input_cost_per_token": 1.923e-06, @@ -13801,7 +13802,8 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { "input_cost_per_token": 1.923e-06, @@ -13810,7 +13812,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { "input_cost_per_token": 1.923e-06, @@ -13819,7 +13822,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/openai/gpt-oss-120b": { "input_cost_per_token": 3.5e-07, @@ -13829,6 +13833,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13839,7 +13844,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-3b-instruct": { "input_cost_per_token": 5.09e-08, @@ -13848,7 +13854,8 @@ "max_output_tokens": 80000, "max_tokens": 80000, "mode": "chat", - "output_cost_per_token": 3.35e-07 + "output_cost_per_token": 3.35e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-guard-3-8b": { "input_cost_per_token": 4.84e-07, @@ -13857,7 +13864,8 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-08 + "output_cost_per_token": 3e-08, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { "input_cost_per_token": 0.0, @@ -13866,7 +13874,8 @@ "max_output_tokens": 15000, "max_tokens": 15000, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -13877,6 +13886,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13888,6 +13898,7 @@ "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, + "rpm": 300, "supports_reasoning": true }, "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { @@ -13897,7 +13908,8 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 2.87e-07 + "output_cost_per_token": 2.87e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-1b-instruct": { "input_cost_per_token": 2.7e-08, @@ -13906,7 +13918,8 @@ "max_output_tokens": 60000, "max_tokens": 60000, "mode": "chat", - "output_cost_per_token": 2.01e-07 + "output_cost_per_token": 2.01e-07, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.6": { "cache_read_input_token_cost": 1.6e-07, @@ -13917,6 +13930,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13928,6 +13942,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13938,7 +13953,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { "input_cost_per_token": 2.93e-07, @@ -13948,6 +13964,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { @@ -13958,6 +13975,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { @@ -13967,7 +13985,8 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "rpm": 300 }, "cloudflare/@cf/zai-org/glm-5.2": { "cache_read_input_token_cost": 2.6e-07, @@ -13978,6 +13997,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13989,6 +14009,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13999,7 +14020,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.55e-07 + "output_cost_per_token": 5.55e-07, + "rpm": 300 }, "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { "input_cost_per_token": 5.09e-08, @@ -14009,6 +14031,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14019,7 +14042,8 @@ "max_output_tokens": 3500, "max_tokens": 3500, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1e-07, @@ -14029,6 +14053,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14040,6 +14065,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { @@ -14050,6 +14076,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, + "rpm": 300, "supports_vision": true }, "cloudflare/@cf/openai/gpt-oss-20b": { @@ -14060,6 +14087,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14071,6 +14099,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwq-32b": { @@ -14081,6 +14110,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, + "rpm": 300, "supports_reasoning": true }, "codestral/codestral-2405": { @@ -54901,6 +54931,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html" + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -60822,6 +60885,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -60832,6 +60896,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", "supported_endpoints": [ "/v1/audio/transcriptions" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8a56a84ade7..cfdbbcb1132 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -54,6 +54,12 @@ from litellm.utils import ( # Adds the parent directory to the system path +def test_cloudflare_model_info_includes_rpm(local_model_cost_map): + assert litellm.get_model_info("cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8")["rpm"] == 300 + assert litellm.get_model_info("cloudflare/@cf/moonshotai/kimi-k2.6")["rpm"] == 20 + assert litellm.get_model_info("cloudflare/@cf/openai/whisper-large-v3-turbo")["rpm"] == 720 + + def test_get_utc_datetime_returns_current_aware_utc_time() -> None: before: Final = datetime.now(timezone.utc) result: Final = litellm.utils.get_utc_datetime() From fd1fad5e0559db253842fdb3972a94081c58734d Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 13:37:56 +0000 Subject: [PATCH 09/81] test: annotate registry metadata test parameters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/test_chatgpt_responses_transformation.py | 6 ++++-- tests/test_litellm/test_utils.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 8a0027fdb53..a7520bd5955 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -54,7 +54,7 @@ class TestChatGPTResponsesAPITransformation: "chatgpt/gpt-5.6-terra", ], ) - def test_chatgpt_responses_model_metadata(self, model_name, local_model_cost_map): + def test_chatgpt_responses_model_metadata(self, model_name: str, local_model_cost_map: None) -> None: model_info = litellm.get_model_info(model_name) assert model_info["litellm_provider"] == "chatgpt" @@ -75,7 +75,9 @@ class TestChatGPTResponsesAPITransformation: "gpt-5.6-terra", ], ) - def test_chatgpt_models_bridge_chat_completions_to_responses(self, model_name, local_model_cost_map): + def test_chatgpt_models_bridge_chat_completions_to_responses( + self, model_name: str, local_model_cost_map: None + ) -> None: """A chat completions request for these models must take the Responses bridge. `gpt-5.6-*` also exists as an openai chat model, so an unregistered diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cfdbbcb1132..45532f91f16 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -54,7 +54,7 @@ from litellm.utils import ( # Adds the parent directory to the system path -def test_cloudflare_model_info_includes_rpm(local_model_cost_map): +def test_cloudflare_model_info_includes_rpm(local_model_cost_map: None) -> None: assert litellm.get_model_info("cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8")["rpm"] == 300 assert litellm.get_model_info("cloudflare/@cf/moonshotai/kimi-k2.6")["rpm"] == 20 assert litellm.get_model_info("cloudflare/@cf/openai/whisper-large-v3-turbo")["rpm"] == 720 From 17dd6afbaafed87e6e7380278ca13a543046160a Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 13:51:58 +0000 Subject: [PATCH 10/81] fix: keep reasoning_effort for mode: responses bridge deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/main.py | 12 +++++++++++- tests/test_litellm/test_main.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 75b7f7f10a5..062ddb9a293 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5398,6 +5398,16 @@ def completion( if dynamic_api_key is not None: api_key = dynamic_api_key # check if user passed in any of the OpenAI optional params + allowed_openai_params: Final[list[str] | None] = cast( + list[str] | None, + ( + [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + if responses_api_model_info.get("mode") == "responses" + and not skip_responses_api_bridge + and "reasoning_effort" not in (kwargs.get("allowed_openai_params") or []) + else kwargs.get("allowed_openai_params") + ), + ) optional_param_args: Final = { "functions": functions, "function_call": function_call, @@ -5442,7 +5452,7 @@ def completion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, - "allowed_openai_params": kwargs.get("allowed_openai_params"), + "allowed_openai_params": allowed_openai_params, "base_model": base_model, } optional_params = get_optional_params(**optional_param_args, **non_default_params) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 038df3656fe..9ce2bb8400d 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1367,6 +1367,40 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( } +@pytest.mark.parametrize("reasoning_effort", ["high", {"effort": "high"}]) +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_responses_bridge_preserves_reasoning_effort_with_drop_params( + mock_responses_completion, reasoning_effort, restore_model_registry +): + mock_responses_completion.return_value = MagicMock() + model = "perplexity/test-responses-bridge" + litellm.register_model( + { + model: { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_reasoning": True, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + with patch.object(litellm, "supports_reasoning", return_value=False): + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=reasoning_effort, + drop_params=True, + api_key="fake-key", + api_base="https://api.perplexity.ai", + ) + + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == reasoning_effort + + @pytest.mark.parametrize( "model, model_info, expected_model_param, expected_base_model_param", [ From 5520c3bb1a768263bbc4458b14da7d87e3696a41 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 13:53:02 +0000 Subject: [PATCH 11/81] refactor: simplify allowed_openai_params bridge override Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/main.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 062ddb9a293..01fe6a7c674 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5398,15 +5398,13 @@ def completion( if dynamic_api_key is not None: api_key = dynamic_api_key # check if user passed in any of the OpenAI optional params - allowed_openai_params: Final[list[str] | None] = cast( - list[str] | None, - ( - [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] - if responses_api_model_info.get("mode") == "responses" - and not skip_responses_api_bridge - and "reasoning_effort" not in (kwargs.get("allowed_openai_params") or []) - else kwargs.get("allowed_openai_params") - ), + bridges_to_responses_api: Final = ( + responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge + ) + allowed_openai_params: Final[list[str] | None] = ( + [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + if bridges_to_responses_api + else kwargs.get("allowed_openai_params") ) optional_param_args: Final = { "functions": functions, From 6f3b3c7957a67d45542538bcda6dc689752b7316 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 14:27:12 +0000 Subject: [PATCH 12/81] test: cover responses bridge reasoning at HTTP boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_main.py | 70 +++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 9ce2bb8400d..8a9d5e07258 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1368,18 +1368,57 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( @pytest.mark.parametrize("reasoning_effort", ["high", {"effort": "high"}]) -@patch("litellm.completion_extras.responses_api_bridge.completion") def test_responses_bridge_preserves_reasoning_effort_with_drop_params( - mock_responses_completion, reasoning_effort, restore_model_registry + reasoning_effort, + restore_model_registry, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ): - mock_responses_completion.return_value = MagicMock() - model = "perplexity/test-responses-bridge" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response_body: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "test-responses-bridge", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + response_route: Final = respx_mock.post("https://api.perplexity.ai/v1/responses").respond(json=response_body) + model: Final = "perplexity/test-responses-bridge" litellm.register_model( { model: { "litellm_provider": "perplexity", "mode": "responses", - "supports_reasoning": True, + "supports_reasoning": False, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, } @@ -1387,18 +1426,17 @@ def test_responses_bridge_preserves_reasoning_effort_with_drop_params( persist_across_reloads=False, ) - with patch.object(litellm, "supports_reasoning", return_value=False): - litellm.completion( - model=model, - messages=[{"role": "user", "content": "hello"}], - reasoning_effort=reasoning_effort, - drop_params=True, - api_key="fake-key", - api_base="https://api.perplexity.ai", - ) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=reasoning_effort, + drop_params=True, + api_key="fake-key", + api_base="https://api.perplexity.ai", + ) - optional_params = mock_responses_completion.call_args.kwargs["optional_params"] - assert optional_params["reasoning_effort"] == reasoning_effort + request_body: Final = json.loads(response_route.calls[0].request.content) + assert request_body["reasoning"] == {"effort": "high"} @pytest.mark.parametrize( From 06497ab42a1ad055e8cbb7695d778cd7a4158ea5 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 15:13:53 +0000 Subject: [PATCH 13/81] fix(ui): guard playground cost metric against null and NaN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_ui/ResponseMetrics.test.tsx | 19 ++++++++++++ .../components/chat_ui/ResponseMetrics.tsx | 2 +- .../llm_calls/chat_completion.test.tsx | 6 ++++ .../components/llm_calls/chat_completion.tsx | 5 +++- .../llm_calls/responses_api.test.tsx | 29 +++++++++++++++++++ .../components/llm_calls/responses_api.tsx | 5 +++- 6 files changed, 63 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx index f31e1839739..4ba6d8d50b4 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx @@ -51,4 +51,23 @@ describe("ResponseMetrics prompt cache chips", () => { expect(screen.queryByText(/Response Cache/)).not.toBeInTheDocument(); }); + + it("does not render the Cost chip when a persisted cost is null", () => { + render(); + + expect(screen.queryByText(/Cost:/)).not.toBeInTheDocument(); + expect(screen.getByText("In: 1")).toBeInTheDocument(); + }); + + it("does not render the Cost chip for NaN", () => { + render(); + + expect(screen.queryByText(/Cost:/)).not.toBeInTheDocument(); + }); + + it("renders the Cost chip for a finite cost", () => { + render(); + + expect(screen.getByText("Cost: $0.000063")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx index ec62d0618d7..bb7debc7aee 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx @@ -159,7 +159,7 @@ const ResponseMetrics: React.FC = ({ timeToFirstToken, tot /> )} - {usage?.cost !== undefined && ( + {typeof usage?.cost === "number" && Number.isFinite(usage.cost) && ( { expect(usageData).not.toHaveProperty("cacheReadTokens"); expect(usageData).not.toHaveProperty("cacheCreationTokens"); }); + + it("omits cost when the provider reports a non-numeric value", async () => { + const usageData = await captureUsage({ cost: "not-a-number" }); + + expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); + }); }); describe("chat_completion response cache", () => { diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index cd0852bc06e..b813a35f687 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -245,7 +245,10 @@ export async function makeOpenAIChatCompletionRequest( // Extract cost from usage object if available if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - usageData.cost = parseFloat(chunkWithUsage.usage.cost); + const parsedCost = parseFloat(chunkWithUsage.usage.cost); + if (Number.isFinite(parsedCost)) { + usageData.cost = parsedCost; + } } onUsageData(usageData); diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 0b94c093acd..7b13f0ac2e3 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -231,6 +231,35 @@ describe("responses_api", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); }); + it("should omit cost when the proxy reports a non-numeric cost", async () => { + async function* streamWithNonNumericCost() { + yield { + type: "response.completed", + response: { + id: "resp_non_numeric_cost", + usage: { output_tokens: 12, input_tokens: 12, total_tokens: 24, cost: "not-a-number" }, + }, + }; + } + mockResponsesCreate.mockResolvedValueOnce(streamWithNonNumericCost()); + + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, + undefined, + undefined, + undefined, + onUsageData, + ); + + expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); + }); + it("should replay MCP output items as events for a non-streaming response", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 7ab76488504..63085a81b94 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -312,7 +312,10 @@ export async function makeOpenAIResponsesRequest( } if (usage.cost !== undefined && usage.cost !== null) { - usageData.cost = Number(usage.cost); + const parsedCost = Number(usage.cost); + if (Number.isFinite(parsedCost)) { + usageData.cost = parsedCost; + } } onUsageData(usageData, mcpToolUsed); From cbe340a31ca81c40be088520ebbaaaca644977ff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:38:18 -0700 Subject: [PATCH 14/81] feat(guardrails): deliver tool-call rewrites into buffered streams A post_call pipeline guardrail that rewrites a streamed tool call (its arguments or its name) now has that rewrite written back across the buffered chunks on chat, Responses, and Messages streams, so the client receives the rewritten tool call instead of the original. The chat handler rewrites the first fragment of each tool-call index and blanks the rest, the Responses handler syncs the function_call output items and their argument events, and the Messages handler rewrites the tool_use content_block_start and input_json_delta events in both dict and SSE-bytes chunks. The delivers_ended_stream_text_rewrites flag becomes delivers_ended_stream_rewrites, since the write-back now covers both text and tool calls, and the executor only discards a tool-call rewrite on translations without write-back or on a shape the translation refuses. --- .../chat/guardrail_translation/handler.py | 176 +++++++++++++++--- .../guardrail_translation/base_translation.py | 17 +- .../chat/guardrail_translation/handler.py | 96 +++++++++- .../guardrail_translation/handler.py | 111 ++++++++++- .../proxy/policy_engine/pipeline_executor.py | 22 +-- litellm/proxy/utils.py | 10 +- .../test_anthropic_guardrail_handler.py | 66 +++++++ .../test_openai_guardrail_handler.py | 73 ++++++++ ...test_openai_responses_guardrail_handler.py | 88 +++++++++ .../policy_engine/test_pipeline_executor.py | 22 ++- .../proxy_logging/test_guardrail_pipeline.py | 9 +- 11 files changed, 618 insertions(+), 72 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e486be12fe2..2049866b444 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,10 +13,11 @@ Pattern Overview: """ import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable from typing_extensions import ReadOnly, TypedDict, assert_never @@ -41,6 +42,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, scoped_structured_message_indices, + stream_item_field, stream_item_fingerprint, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( @@ -153,6 +155,28 @@ class ExtractedInput: EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=()) +@dataclass(frozen=True, slots=True) +class _ToolCallShape: + name: str | None + arguments: str + + +_SSEEventRewriter = Callable[[Mapping[str, object]], Mapping[str, object] | None] + + +def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]: + """The guardrail-visible shape of each tool call, whether the guardrail handed + back the ``ChatCompletionMessageToolCall`` objects it was given or plain dicts.""" + functions: Final = tuple(stream_item_field(tool_call, "function") for tool_call in tool_calls) + return tuple( + _ToolCallShape( + name=name if isinstance(name := stream_item_field(function, "name"), str) else None, + arguments=arguments if isinstance(arguments := stream_item_field(function, "arguments"), str) else "", + ) + for function in functions + ) + + class _AnthropicSSEDelta(TypedDict, total=False): type: ReadOnly[str] text: ReadOnly[str] @@ -170,7 +194,7 @@ class AnthropicMessagesHandler(BaseTranslation): them through guardrail rewrites; downstream provider handling is out of scope. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def __init__(self): super().__init__() @@ -1050,6 +1074,7 @@ class AnthropicMessagesHandler(BaseTranslation): first_choice.message.tool_calls, ) string_so_far = first_choice.message.content + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) guardrail_inputs: Final = GenericGuardrailAPIInputs() if string_so_far: guardrail_inputs["texts"] = [string_so_far] @@ -1084,6 +1109,19 @@ class AnthropicMessagesHandler(BaseTranslation): and guardrailed_texts[0] != string_so_far ): self._write_ended_stream_text_rewrite(responses_so_far, guardrailed_texts[0]) + if deliver_ended_stream_rewrites: + returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls") + self._write_ended_stream_tool_call_rewrites( + responses_so_far, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=_tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(pre_guardrail_tool_calls) + else tool_calls_list or () + ), + guardrail_name=guardrail_to_apply.guardrail_name or "unknown", + ) else: verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices") return responses_so_far @@ -1212,38 +1250,120 @@ class AnthropicMessagesHandler(BaseTranslation): """Deliver an ended-stream guardrail text rewrite by rewriting the buffered chunks in place: the first ``text_delta`` carries the full rewritten text and every later one is blanked, leaving the surrounding - message and content-block framing untouched. Handles both chunk formats - this stream carries (parsed event dicts and raw SSE bytes).""" + message and content-block framing untouched.""" replacements: Final = chain((rewritten_text,), repeat("")) - for idx, item in enumerate(responses_so_far): - if isinstance(item, dict): - delta = item.get("delta") - if item.get("type") == "content_block_delta" and isinstance(delta, dict): - if delta.get("type") == "text_delta": - delta["text"] = next(replacements) - elif isinstance(item, (bytes, bytearray)): - responses_so_far[idx] = ( # rebind-ok: delivers the rewrite into the caller's buffer - AnthropicMessagesHandler._rewrite_sse_text_deltas(bytes(item), replacements) - ) + + def rewrite_text_delta(event: Mapping[str, object]) -> Mapping[str, object] | None: + delta: Final = event.get("delta") + if event.get("type") != "content_block_delta" or not isinstance(delta, Mapping): + return None + if delta.get("type") != "text_delta": + return None + return {**event, "delta": {**delta, "text": next(replacements)}} + + AnthropicMessagesHandler._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta) + + @classmethod + def _write_ended_stream_tool_call_rewrites( + cls, + responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + *, + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str, + ) -> None: + """Deliver ended-stream guardrail tool-call rewrites by rewriting the + buffered chunks in place: the rebuilt response lists tool calls in the + order of the stream's ``tool_use`` blocks, so the nth rewritten call lands + on the nth block, its first ``input_json_delta`` carrying the full rewritten + arguments, every later one blanked, and ``content_block_start`` carrying the + rewritten name. Blocks that do not line up with the rebuilt tool calls make + the rewrite undeliverable, so the pipeline executor discards it and releases + the original chunks.""" + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + block_indices: Final = tuple( + index + for item in responses_so_far + for event in cls._iter_sse_events(item) + if event.get("type") == "content_block_start" + and isinstance(block := event.get("content_block"), Mapping) + and block.get("type") == "tool_use" + and isinstance(index := event.get("index"), int) + ) + if len(block_indices) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + rewrites_by_block: Final = MappingProxyType( + { + index: after + for index, before, after in zip(block_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls) + if after != before + } + ) + argument_replacements: Final = MappingProxyType( + {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_block.items()} + ) + + def rewrite_tool_use(event: Mapping[str, object]) -> Mapping[str, object] | None: + index: Final = event.get("index") + if not isinstance(index, int) or index not in rewrites_by_block: + return None + match event.get("type"): + case "content_block_start": + block: Final = event.get("content_block") + name: Final = rewrites_by_block[index].name + if not isinstance(block, Mapping) or name is None: + return None + return {**event, "content_block": {**block, "name": name}} + case "content_block_delta": + delta: Final = event.get("delta") + if not isinstance(delta, Mapping) or delta.get("type") != "input_json_delta": + return None + return {**event, "delta": {**delta, "partial_json": next(argument_replacements[index])}} + case _: + return None + + cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use) @staticmethod - def _rewrite_sse_text_deltas(sse_bytes: bytes, replacements: "Iterator[str]") -> bytes: - """Rewrite every ``text_delta`` data line in one SSE chunk with the next - replacement text, leaving all other events and framing byte-identical.""" + def _rewrite_ended_stream_events( + responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + rewrite_event: _SSEEventRewriter, + ) -> None: + """Replace every buffered event ``rewrite_event`` returns a rewrite for, in + both chunk formats this stream carries (parsed event dicts and raw SSE + bytes), leaving every other event and the framing untouched.""" + rewritten_items: Final = tuple( + AnthropicMessagesHandler._rewrite_buffered_item(item, rewrite_event) for item in responses_so_far + ) + responses_so_far[:] = rewritten_items # rebind-ok: delivers the rewrites into the caller's buffer + + @staticmethod + def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: + if isinstance(item, dict): + rewritten: Final = rewrite_event(_as_str_mapping(item)) + return item if rewritten is None else dict(rewritten) + if isinstance(item, (bytes, bytearray)): + return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) + return item + + @staticmethod + def _rewrite_sse_events(sse_bytes: bytes, rewrite_event: _SSEEventRewriter) -> bytes: + """Rewrite the data lines of one SSE chunk that ``rewrite_event`` rewrites, + leaving all other events and framing byte-identical.""" try: decoded: Final = sse_bytes.decode("utf-8") except UnicodeDecodeError: return sse_bytes return "\n\n".join( - AnthropicMessagesHandler._rewrite_sse_block(block, replacements) for block in decoded.split("\n\n") + "\n".join(AnthropicMessagesHandler._rewrite_sse_line(line, rewrite_event) for line in block.split("\n")) + for block in decoded.split("\n\n") ).encode("utf-8") @staticmethod - def _rewrite_sse_block(block: str, replacements: "Iterator[str]") -> str: - return "\n".join(AnthropicMessagesHandler._rewrite_sse_line(line, replacements) for line in block.split("\n")) - - @staticmethod - def _rewrite_sse_line(line: str, replacements: "Iterator[str]") -> str: + def _rewrite_sse_line(line: str, rewrite_event: _SSEEventRewriter) -> str: if not line.startswith("data:"): return line try: @@ -1252,14 +1372,10 @@ class AnthropicMessagesHandler(BaseTranslation): ) except json.JSONDecodeError: return line - if not isinstance(data, dict) or data.get("type") != "content_block_delta": + if not isinstance(data, dict): return line - delta: Final = data.get("delta") - if not isinstance(delta, dict) or delta.get("type") != "text_delta": - return line - return "data: " + json.dumps( - {**data, "delta": {**delta, "text": next(replacements)}} # mutable-ok: json.dumps needs plain dicts - ) + rewritten: Final = rewrite_event(_as_str_mapping(data)) + return line if rewritten is None else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index afd8e0f67f7..6d1a9ab1c3e 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -52,13 +52,14 @@ class StreamingScanKey: class BaseTranslation(ABC): - delivers_ended_stream_text_rewrites: ClassVar[bool] = False + delivers_ended_stream_rewrites: ClassVar[bool] = False """Whether ``process_output_streaming_response`` accepts ``deliver_ended_stream_rewrites=True`` and, on an ended (fully buffered) - stream, writes guardrail text rewrites back across ``responses_so_far`` so - a buffered pipeline can release rewritten chunks. Tool-call rewrites, and - text rewrites on every other translation, are undeliverable: the pipeline - executor discards them and releases the original chunks.""" + stream, writes guardrail text and tool-call rewrites back across + ``responses_so_far`` so a buffered pipeline can release rewritten chunks, + raising ``UndeliverableStreamRewrite`` for a shape it cannot place. Rewrites + on every other translation are undeliverable: the pipeline executor + discards them and releases the original chunks.""" @staticmethod def transform_user_api_key_dict_to_metadata( @@ -175,9 +176,9 @@ class BaseTranslation(ABC): transformations (see ``StreamTransformSink``); base handlers ignore it. ``deliver_ended_stream_rewrites`` is passed True only when the caller holds the whole buffered stream and the subclass declares - ``delivers_ended_stream_text_rewrites``: the handler then writes - guardrail text rewrites back across ``responses_so_far`` instead of - discarding them. + ``delivers_ended_stream_rewrites``: the handler then writes + guardrail text and tool-call rewrites back across ``responses_so_far`` + instead of discarding them. """ return responses_so_far diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 80292aef2cf..42c95ac3316 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -49,6 +49,8 @@ from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import coerce_stream_holdback_value, ) from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + ChatCompletionMessageToolCall, Choices, GenericGuardrailAPIInputs, ModelResponse, @@ -78,7 +80,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Methods can be overridden to customize behavior for different message formats. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ @@ -610,13 +612,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): deliver_ended_stream_rewrites: bool, ) -> None: """Ended-stream path: rebuild the full response, run the non-streaming - output guardrail against it, and (when opted in) write any text rewrite - back across the buffered chunks.""" + output guardrail against it, and (when opted in) write any text or + tool-call rewrite back across the buffered chunks.""" model_response: Final = cast( ModelResponse, stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj), ) pre_guardrail_texts: Final = self._string_choice_contents(model_response) + pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) await self.process_output_response( response=model_response, guardrail_to_apply=guardrail_to_apply, @@ -624,13 +627,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation): user_api_key_dict=user_api_key_dict, request_data=request_data, ) - if deliver_ended_stream_rewrites: - await self._write_ended_stream_text_rewrites( - responses_so_far=responses_so_far, - guardrailed_response=model_response, - pre_guardrail_texts=pre_guardrail_texts, - guardrail_name=guardrail_to_apply.guardrail_name or "unknown", - ) + if not deliver_ended_stream_rewrites: + return + guardrail_name: Final = guardrail_to_apply.guardrail_name or "unknown" + await self._write_ended_stream_text_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + pre_guardrail_texts=pre_guardrail_texts, + guardrail_name=guardrail_name, + ) + self._write_ended_stream_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_name, + ) def build_stream_error_items( self, @@ -1043,6 +1054,71 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=[(target_choice_index, None) for _ in changed], # mutable-ok: callee takes lists ) + @staticmethod + def _function_tool_call_shapes(response: "ModelResponse") -> tuple[tuple[str | None, str], ...]: + return tuple( + (tool_call.function.name, tool_call.function.arguments) + for choice in response.choices + for tool_call in choice.message.tool_calls or () + if isinstance(tool_call, ChatCompletionMessageToolCall) + ) + + @staticmethod + def _function_tool_call_fragments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: + """Group the stream's function tool-call fragments by their tool-call index, in + the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping + only the indices the builder keeps (an id and a name somewhere in the stream).""" + fragments: Final = tuple( + tool_call + for response in responses_so_far + for choice in response.choices + for tool_call in choice.delta.tool_calls or () + if isinstance(tool_call, ChatCompletionDeltaToolCall) + ) + identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) + named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + return tuple( + tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + ) + + def _write_ended_stream_tool_call_rewrites( + self, + responses_so_far: list["ModelResponseStream"], # mutable-ok: rewrites the caller's buffered chunks in place + guardrailed_response: "ModelResponse", + pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], + guardrail_name: str, + ) -> None: + """Write ended-stream guardrail tool-call rewrites back across the buffered + chunks: the rewritten name and full arguments land in the tool call's first + fragment and the arguments of its later fragments are blanked, mirroring the + text write-back. A rewrite on a stream carrying more than one distinct choice + index, or whose fragments do not line up with the rebuilt tool calls, is + reported as undeliverable, so the pipeline executor discards it and releases + the original chunks.""" + post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + stream_choice_indices: Final = frozenset( + choice.index for response in responses_so_far for choice in response.choices + ) + fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) + if len(stream_choice_indices) != 1 or len(fragments_by_tool_call) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + for before, (name, arguments), fragments in zip( + pre_guardrail_tool_calls, post_guardrail_tool_calls, fragments_by_tool_call + ): + if (name, arguments) == before: + continue + head, *tail = fragments + head.function.name = name + head.function.arguments = arguments + for fragment in tail: + fragment.function.arguments = "" + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index b0f79552bc5..447870fc0be 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -101,6 +101,18 @@ if TYPE_CHECKING: from litellm.types.llms.openai import ResponseInputParam +class _ToolCallShape(NamedTuple): + name: str | None + arguments: str + + +def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tuple[_ToolCallShape, ...]: + return tuple( + _ToolCallShape(name=tool_call["function"].get("name"), arguments=tool_call["function"].get("arguments", "")) + for tool_call in tool_calls + ) + + class ResponseOutputEnvelope(TypedDict, total=False): """Dict form of a Responses API response, as far as guardrail write-back reads it.""" @@ -340,7 +352,7 @@ class OpenAIResponsesHandler(BaseTranslation): Methods can be overridden to customize behavior for different message formats. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ @@ -754,6 +766,7 @@ class OpenAIResponsesHandler(BaseTranslation): if response_model: inputs["model"] = response_model + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=request_data, @@ -762,6 +775,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") + post_guardrail_tool_calls: Final = _tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) + else tool_calls_to_check + ) # Write guardrailed texts back into the output items in-place. # final_chunk is a reference into responses_so_far so this @@ -784,6 +803,13 @@ class OpenAIResponsesHandler(BaseTranslation): stream_events=responses_so_far[:-1], rewrites_by_position=rewrites_by_position, ) + self._deliver_ended_stream_tool_call_rewrites( + responses_so_far=responses_so_far, + outputs=outputs, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=post_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name or "unknown", + ) return responses_so_far # ------------------------------------------------------------------ # @@ -894,6 +920,89 @@ class OpenAIResponsesHandler(BaseTranslation): continue OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten) + def _deliver_ended_stream_tool_call_rewrites( + self, + responses_so_far: Sequence[object], + outputs: Sequence[object], + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str, + ) -> None: + """Write ended-stream guardrail tool-call rewrites into the completed + envelope's ``function_call`` items and sync the earlier stream events, + keyed by ``output_index``. The guardrail sees the envelope's function + calls in output order, which is how a rewritten call finds its item; a + rewrite whose calls do not line up with the envelope is reported as + undeliverable, so the pipeline executor discards it and releases the + original events.""" + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + function_call_indices: Final = tuple( + output_idx + for output_idx, output_item in enumerate(outputs) + if stream_item_field(output_item, "type") == "function_call" + ) + if len(function_call_indices) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + rewrites_by_output_index: Final = MappingProxyType( + { + output_idx: after + for output_idx, before, after in zip( + function_call_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls + ) + if after != before + } + ) + for output_idx, rewrite in rewrites_by_output_index.items(): + self._write_function_call_item(outputs[output_idx], rewrite.name, rewrite.arguments) + self._sync_stream_events_with_tool_call_rewrites( + stream_events=responses_so_far[:-1], + rewrites_by_output_index=rewrites_by_output_index, + ) + + def _sync_stream_events_with_tool_call_rewrites( + self, + stream_events: Sequence[object], + rewrites_by_output_index: Mapping[int, _ToolCallShape], + ) -> None: + """Sync pre-completion function-call events with the rewritten completed + response: the first ``function_call_arguments.delta`` for a rewritten call + carries the full rewritten arguments and the rest are blanked, while + ``function_call_arguments.done`` and ``output_item.done`` carry the full + rewritten arguments and ``output_item.added`` / ``output_item.done`` the + rewritten name, so every event a client may read agrees with the + rewritten ``response.completed`` payload.""" + delta_replacements: Final = MappingProxyType( + {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_output_index.items()} + ) + for event in stream_events: + output_index = stream_item_field(event, "output_index") + if not isinstance(output_index, int) or output_index not in rewrites_by_output_index: + continue + rewrite = rewrites_by_output_index[output_index] + match stream_item_field(event, "type"): + case "response.function_call_arguments.delta": + self._write_event_field(event, "delta", next(delta_replacements[output_index])) + case "response.function_call_arguments.done": + self._write_event_field(event, "arguments", rewrite.arguments) + case "response.output_item.added": + self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, None) + case "response.output_item.done": + self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, rewrite.arguments) + case _: + pass + + @staticmethod + def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: + if not (isinstance(item, dict) or hasattr(item, "get")): + return + if name is not None: + OpenAIResponsesHandler._write_event_field(item, "name", name) + if arguments is not None: + OpenAIResponsesHandler._write_event_field(item, "arguments", arguments) + def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ Check if the streaming has ended. diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9bc10949e9f..3e112bf8e67 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -85,10 +85,10 @@ def _logged_by_inner_guardrail(method: _GuardrailMethodT) -> _GuardrailMethodT: class _StreamRewriteObserver(CustomGuardrail): """Stand-in handed to the endpoint translation in place of a streaming pipeline step's guardrail. It records whether the guardrail returned different output than it was given, - which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text - rewrites are deliverable on translations that write them back across the buffered chunks - (``delivers_ended_stream_text_rewrites``); tool-call rewrites and text rewrites on any - other translation are discarded by the executor, which releases the original chunks. + which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text and + tool-call rewrites are deliverable on translations that write them back across the + buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation + are discarded by the executor, which releases the original chunks. The inner guardrail's ``apply_guardrail`` already records the guardrail information and span, so the observer's stays out of ``log_guardrail_information``.""" @@ -290,13 +290,13 @@ class PipelineExecutor: litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> None: """Run one streaming post_call step through the endpoint translation, delivering - text rewrites on translations that support ended-stream write-back. A rewrite that - cannot reach the client yet (a tool-call rewrite, a text rewrite on a translation - without write-back, or one the translation refused with - ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the - originals and the step passes, so the client gets the stream the merge base sent.""" + text and tool-call rewrites on translations that support ended-stream write-back. A + rewrite that cannot reach the client yet (one on a translation without write-back, or + one the translation refused with ``UndeliverableStreamRewrite``) is discarded: the + buffered chunks go back to the originals and the step passes, so the client gets the + stream the merge base sent.""" observer: Final = _StreamRewriteObserver(callback) - deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites + deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_rewrites originals: Final = copy.deepcopy(streaming_chunks) try: if deliver_rewrites: @@ -319,7 +319,7 @@ class PipelineExecutor: except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) else: - if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): + if not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls): _release_original_chunks(step.guardrail, streaming_chunks, originals) if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8b36061c6be..76b9b4b44ed 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3445,11 +3445,11 @@ class ProxyLogging: assembled output through the endpoint guardrail translation, the same machinery flat post_call guardrails use at end of stream. An allow releases the buffered chunks: verbatim when no guardrail rewrote the - output, rewritten in place when one rewrote text and the translation - delivers ended-stream rewrites (later steps then re-scan the rewritten - chunks, so rewrites chain). A rewrite the translation cannot deliver - yet (a tool-call rewrite, or a text rewrite on a route without - write-back) is discarded by the executor and the original chunks are + output, rewritten in place when one rewrote text or a tool call and the + translation delivers ended-stream rewrites (later steps then re-scan the + rewritten chunks, so rewrites chain). A rewrite the translation cannot + deliver yet (one on a route without write-back, or a shape the route + refuses) is discarded by the executor and the original chunks are released, as is a buffered shape no translation resolves; a block or modify_response terminates with the translation's block chunks or the raised error. diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bf40f781fa3..bd2dff9b33c 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -315,6 +315,72 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert "event: message_start" in raw and "event: message_stop" in raw assert '"stop_reason": "end_turn"' in raw + @staticmethod + def _ended_tool_use_sse_chunks() -> list: + events = [ + ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-sonnet-4-5", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup_fruit", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"fruit": "persim'}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": 'mon"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 2}}), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + @staticmethod + def _argument_masking_guardrail() -> CustomGuardrail: + class MaskArguments(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call.function.arguments = '{"fruit": "[MASKED]"}' + return inputs + + return MaskArguments(guardrail_name="test") + + @staticmethod + def _partial_jsons(chunks: list) -> list: + return [ + json.loads(line[len("data:") :].strip())["delta"]["partial_json"] + for chunk in chunks + for line in chunk.decode().split("\n") + if line.startswith("data:") and json.loads(line[len("data:") :].strip()).get("type") == "content_block_delta" + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_use_input_back_into_sse_chunks(self): + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + + result = await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + assert result is chunks + assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', "", ""] + raw = b"".join(chunks).decode() + assert '"name": "lookup_fruit"' in raw and '"id": "toolu_1"' in raw + assert '"stop_reason": "tool_use"' in raw + assert "persim" not in raw + + @pytest.mark.asyncio + async def test_ended_stream_tool_use_rewrite_leaves_chunks_untouched_by_default(self): + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + original = [bytes(chunk) for chunk in chunks] + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert chunks == original + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index aff0530ee8b..be168ec83e6 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1113,6 +1113,79 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[1].choices[0].delta.content in (None, "") assert chunks[1].choices[0].finish_reason == "stop" + @staticmethod + def _ended_tool_call_stream_chunks() -> list: + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + def chunk(tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None): + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(tool_calls=[tool_call] if tool_call else None), + finish_reason=finish_reason, + ) + ], + ) + + def fragment(arguments: str, name: Optional[str] = None, call_id: Optional[str] = None): + return ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments) + ) + + return [ + chunk(fragment("", name="lookup_fruit", call_id="call_1")), + chunk(fragment('{"fruit":')), + chunk(fragment(' "persimmon"}')), + chunk(None, finish_reason="tool_calls"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="test") + chunks = self._ended_tool_call_stream_chunks() + + result = await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is chunks + fragments = [chunk.choices[0].delta.tool_calls for chunk in chunks[:3]] + assert [fragment[0].function.arguments for fragment in fragments] == ['{"fruit": "PERSIMMON"}', "", ""] + assert fragments[0][0].function.name == "lookup_fruit" + assert fragments[0][0].id == "call_1" + assert chunks[3].choices[0].delta.tool_calls is None + assert chunks[3].choices[0].finish_reason == "tool_calls" + + @pytest.mark.asyncio + async def test_ended_stream_tool_call_rewrite_leaves_chunks_untouched_by_default(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="test") + chunks = self._ended_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + ) + + fragments = [chunk.choices[0].delta.tool_calls for chunk in chunks[:3]] + assert [fragment[0].function.arguments for fragment in fragments] == ["", '{"fruit":', ' "persimmon"}'] + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 33c8c97fea7..a024fc7ff81 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1195,6 +1195,94 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]" assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]" + @staticmethod + def _ended_function_call_stream_events() -> List[dict]: + def item(arguments: str, status: str) -> dict: + return { + "type": "function_call", + "id": "fc_123", + "call_id": "call_123", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_123", "output_index": 0, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_123", "output_index": 0, "delta": ' "persimmon"}'}, + { + "type": "response.function_call_arguments.done", + "item_id": "fc_123", + "output_index": 0, + "arguments": '{"fruit": "persimmon"}', + }, + {"type": "response.output_item.done", "output_index": 0, "item": item('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "model": "gpt-4o", + "output": [item('{"fruit": "persimmon"}', "completed")], + "status": "completed", + }, + }, + ] + + @staticmethod + def _argument_masking_guardrail() -> CustomGuardrail: + class MaskArguments(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + tool_calls = [ + {**tool_call, "function": {**tool_call["function"], "arguments": '{"fruit": "[MASKED]"}'}} + for tool_call in inputs.get("tool_calls", []) + ] + return {**inputs, "tool_calls": tool_calls} + + return MaskArguments(guardrail_name="test-mask-arguments") + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_function_call_events(self): + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + + result = await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is events + assert events[0]["item"]["arguments"] == "" + assert events[1]["delta"] == '{"fruit": "[MASKED]"}' + assert events[2]["delta"] == "" + assert events[3]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["response"]["output"][0]["name"] == "lookup_fruit" + + @pytest.mark.asyncio + async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + ) + + assert events[1]["delta"] == '{"fruit":' + assert events[3]["arguments"] == '{"fruit": "persimmon"}' + assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "persimmon"}' + @pytest.mark.asyncio @pytest.mark.parametrize("terminal_type", ["response.incomplete", "response.failed"]) async def test_deliver_ended_stream_rewrites_syncs_non_completed_terminals(self, terminal_type): diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 54ef1f79f4d..ee75003db9f 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -924,7 +924,7 @@ class _TextReturningGuardrail(CustomGuardrail): class _TextTranslation: - delivers_ended_stream_text_rewrites = False + delivers_ended_stream_rewrites = False def __init__(self): self.seen_guardrail_names = [] @@ -946,7 +946,7 @@ class _WritingTranslation: """Writes the guardrail's text (and tool-call) outputs back into the buffered chunks the way the chat/Responses/Messages handlers do on an ended stream.""" - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True async def process_output_streaming_response( self, @@ -970,7 +970,7 @@ class _WritingTranslation: class _RefusingTranslation: - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True async def process_output_streaming_response( self, @@ -1088,13 +1088,27 @@ async def test_streaming_step_delivers_text_rewrite_through_writing_translation( @pytest.mark.asyncio -async def test_streaming_step_discards_tool_call_rewrite_and_restores_written_text(monkeypatch, caplog): +async def test_streaming_step_delivers_tool_call_rewrite_through_writing_translation(monkeypatch, caplog): monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): result = await _run_streaming_step(_WritingTranslation(), chunks) + assert result.terminal_action == "allow" + assert chunks[0]["text"] == "hello [MASKED]" + assert chunks[0]["tool_call"]["function"]["arguments"] == '{"ssn": "[MASKED]"}' + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_tool_call_rewrite_when_translation_lacks_write_back(monkeypatch, caplog): + monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_streaming_step(_TextTranslation(), chunks) + _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 73dd6746b29..d27f504ba6d 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1836,7 +1836,7 @@ def _echoed_tool_call_dicts(arguments: str) -> List[Dict[str, Any]]: @pytest.mark.asyncio @pytest.mark.parametrize("on_fail, on_error", [("block", None), ("next", "next")]) -async def test_streaming_iterator_hook_pipeline_releases_originals_on_runtime_tool_call_rewrite( +async def test_streaming_iterator_hook_pipeline_delivers_runtime_tool_call_rewrite( proxy_logging, make_user_api_key_auth, monkeypatch, on_fail, on_error, caplog ): transform = lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')} # noqa: E731 @@ -1855,9 +1855,12 @@ async def test_streaming_iterator_hook_pipeline_releases_originals_on_runtime_to delivered.append(item) assert len(delivered) == 2 - assert delivered[0].choices[0].delta.tool_calls[0].function.arguments == '{"ssn": "123"}' + delivered_tool_call = delivered[0].choices[0].delta.tool_calls[0] + assert delivered_tool_call.function.arguments == '{"ssn": "[MASKED]"}' + assert delivered_tool_call.function.name == "lookup" + assert delivered_tool_call.id == "call_1" assert delivered[1].choices[0].finish_reason == "tool_calls" - assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + assert not any("discarded" in message for message in _warnings(caplog)) @pytest.mark.asyncio From e65c56876f9aae1f94120212a2092592792f6538 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:42:51 -0700 Subject: [PATCH 15/81] fix(guardrails): write tool-call rewrites into typed Responses envelopes The response.completed envelope carries its function_call output items as SDK objects without a get shim, so the write-back skipped them and the envelope still showed the original arguments after every stream event had been rewritten. Write the item whenever one is present, and cover the typed event shape the live proxy carries in the handler test. --- .../guardrail_translation/handler.py | 2 +- ...test_openai_responses_guardrail_handler.py | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 447870fc0be..208ceb959e6 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -996,7 +996,7 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: - if not (isinstance(item, dict) or hasattr(item, "get")): + if item is None: return if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index a024fc7ff81..bdf7f83757c 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1222,6 +1222,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: "type": "response.completed", "response": { "id": "resp_123", + "created_at": 1, "model": "gpt-4o", "output": [item('{"fruit": "persimmon"}', "completed")], "status": "completed", @@ -1268,6 +1269,51 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' assert events[5]["response"]["output"][0]["name"] == "lookup_fruit" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_typed_function_call_events(self): + from litellm.types.llms.openai import ( + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ) + + handler = OpenAIResponsesHandler() + typed_events: List[Any] = [ + model.model_validate(event) + for model, event in zip( + ( + OutputItemAddedEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ), + self._ended_function_call_stream_events(), + ) + ] + completed_event = typed_events[5] + assert isinstance(completed_event, ResponseCompletedEvent) + assert isinstance(completed_event.response, ResponsesAPIResponse) + assert isinstance(completed_event.response.output[0], ResponseFunctionToolCall) + + await handler.process_output_streaming_response( + responses_so_far=typed_events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert typed_events[1].delta == '{"fruit": "[MASKED]"}' + assert typed_events[2].delta == "" + assert typed_events[3].arguments == '{"fruit": "[MASKED]"}' + assert typed_events[4].item.arguments == '{"fruit": "[MASKED]"}' + assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' + assert completed_event.response.output[0].name == "lookup_fruit" + @pytest.mark.asyncio async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): handler = OpenAIResponsesHandler() From c6f5763443504501e88113e783d2e1ec88b9264d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:04:52 -0700 Subject: [PATCH 16/81] feat(guardrails): run legacy post-call hooks as streaming pipeline steps A post_call pipeline step whose guardrail only implements the older async_post_call_success_hook used to skip the stream entirely: PR #38721 fails that shape open with a warning. The streaming step now assembles the buffered stream into the response the hook expects, runs the hook, ends the stream with the hook's exception when it raises, and delivers the hook's rewrite through the same event write-back the unified guardrails use on chat, Responses, and Messages streams (Messages gets the Anthropic shape). A stream a pipeline manages no longer runs the same hook again after the stream ends. A guardrail with neither the unified interface nor a post-call hook keeps the fail-open, as does a rewrite the buffer cannot be patched with. --- .../chat/guardrail_translation/handler.py | 5 + .../guardrail_translation/base_translation.py | 7 + litellm/proxy/common_request_processing.py | 15 +- .../proxy/policy_engine/pipeline_executor.py | 113 ++++++++- litellm/proxy/utils.py | 30 +-- .../test_anthropic_guardrail_handler.py | 26 +++ .../policy_engine/test_pipeline_executor.py | 150 +++++++++++- .../test_proxy_logging_hook_detection.py | 26 +++ .../proxy_logging/test_guardrail_pipeline.py | 220 +++++++++++++----- 9 files changed, 505 insertions(+), 87 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e486be12fe2..f61cfe58e80 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -176,6 +176,11 @@ class AnthropicMessagesHandler(BaseTranslation): super().__init__() self.adapter = LiteLLMAnthropicMessagesAdapter() + def post_call_hook_response(self, response: object) -> object: + if not isinstance(response, ModelResponse): + return response + return self.adapter.translate_openai_response_to_anthropic(response) + @staticmethod def _build_streaming_usage_response( responses_so_far: Sequence[object], diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index afd8e0f67f7..bef472b2882 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -60,6 +60,13 @@ class BaseTranslation(ABC): text rewrites on every other translation, are undeliverable: the pipeline executor discards them and releases the original chunks.""" + def post_call_hook_response(self, response: object) -> object: + """The ``response`` this endpoint's non-streaming post-call hooks receive, derived from + the object the translation stores under ``request_data["response"]`` while scanning an + ended stream. Chat and Responses scan that shape already; a translation that scans a + different one (Messages scans an OpenAI-shaped ModelResponse) overrides this.""" + return response + @staticmethod def transform_user_api_key_dict_to_metadata( user_api_key_dict: Any | None, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9720e4b1cf8..7e85b66682c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3328,9 +3328,10 @@ class ProxyBaseLLMRequestProcessing: has completed. Guardrails routed through unified_guardrail are skipped, since they already ran - via its streaming iterator. Guardrails that override - async_post_call_success_hook directly run here, including those that implement - apply_guardrail but keep their native lifecycle hooks. + via its streaming iterator, and so are guardrails a post_call policy pipeline + manages, since the pipeline ran them against the buffered stream. Guardrails + that override async_post_call_success_hook directly run here, including those + that implement apply_guardrail but keep their native lifecycle hooks. This is audit-only — content has already been delivered to the client. @@ -3340,12 +3341,18 @@ class ProxyBaseLLMRequestProcessing: _response = assembled_response try: from litellm.proxy.proxy_server import llm_router as _global_llm_router - from litellm.proxy.utils import _check_and_merge_model_level_guardrails + from litellm.proxy.utils import ( + _check_and_merge_model_level_guardrails, + pipeline_managed_guardrail_names, + ) guardrail_data = _check_and_merge_model_level_guardrails(data=captured_data, llm_router=_global_llm_router) + pipeline_managed: Final = pipeline_managed_guardrail_names(captured_data, "post_call") for cb in litellm.callbacks: if not isinstance(cb, CustomGuardrail): continue + if cb.guardrail_name in pipeline_managed: + continue if not cb.should_run_guardrail( data=guardrail_data, event_type=GuardrailEventHooks.post_call, diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9bc10949e9f..25c54d72f25 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -121,6 +121,80 @@ class _StreamRewriteObserver(CustomGuardrail): return outputs +class _ScannedTextRecorder(CustomGuardrail): + def __init__(self, guardrail_name: str) -> None: + super().__init__(guardrail_name=guardrail_name) + self.texts: tuple[str, ...] | None = None + + @_logged_by_inner_guardrail + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, # mutable-ok: matches CustomGuardrail.apply_guardrail + input_type: Literal["request", "response"], + logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> GenericGuardrailAPIInputs: + self.texts = _text_snapshot(inputs.get("texts")) + return inputs + + +class _LegacyHookStreamAdapter(CustomGuardrail): + """Runs a guardrail that only implements the legacy post-call hook (no unified + ``apply_guardrail``, or ``use_native_lifecycle_hooks``) as a streaming pipeline step. The + endpoint translation hands it the texts it scanned plus the assembled response under + ``request_data["response"]``; the hook gets that response in the shape its route gives + non-streaming hooks, an exception it raises ends the stream through the executor's + fail/error classification, and a replacement response is re-scanned by the same translation + so its texts reach the client through the translation's ended-stream write-back. A + replacement whose scanned texts do not line up with the originals is undeliverable, so the + executor releases the original chunks.""" + + def __init__( + self, + inner: CustomGuardrail, + endpoint_translation: "BaseTranslation", + user_api_key_dict: "UserAPIKeyAuth", + ) -> None: + super().__init__(guardrail_name=inner.guardrail_name) + self.inner: Final = inner + self.endpoint_translation: Final = endpoint_translation + self.user_api_key_dict: Final = user_api_key_dict + + def structured_messages_cover_full_request(self) -> bool: + return self.inner.structured_messages_cover_full_request() + + @_logged_by_inner_guardrail + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, # mutable-ok: matches CustomGuardrail.apply_guardrail + input_type: Literal["request", "response"], + logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> GenericGuardrailAPIInputs: + replacement: Final = await self.inner.async_post_call_success_hook( + data=request_data, + user_api_key_dict=self.user_api_key_dict, + response=self.endpoint_translation.post_call_hook_response(request_data.get("response")), + ) + if replacement is None: + return inputs + scanned: Final = _text_snapshot(inputs.get("texts")) + rewritten: Final = await self._scanned_texts(replacement, logging_obj) + if scanned is None or rewritten is None or len(rewritten) != len(scanned): + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + return {**inputs, "texts": list(rewritten)} + + async def _scanned_texts(self, response: object, logging_obj: "LiteLLMLoggingObj | None") -> tuple[str, ...] | None: + recorder: Final = _ScannedTextRecorder(self.guardrail_name or "unknown") + await self.endpoint_translation.process_output_response( + response=response, + guardrail_to_apply=recorder, + litellm_logging_obj=logging_obj, + user_api_key_dict=self.user_api_key_dict, + ) + return recorder.texts + + def _prepare_hook_input( step: PipelineStep, callback: CustomGuardrail, @@ -286,16 +360,23 @@ class PipelineExecutor: endpoint_translation: "BaseTranslation", streaming_chunks: list[object], # mutable-ok: shared buffered-stream chunks the translation rewrites in place hook_input: dict[str, object], # mutable-ok: same request-payload shape as data - user_api_key_dict: "UserAPIKeyAuth | None", + user_api_key_dict: "UserAPIKeyAuth", litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> None: """Run one streaming post_call step through the endpoint translation, delivering - text rewrites on translations that support ended-stream write-back. A rewrite that - cannot reach the client yet (a tool-call rewrite, a text rewrite on a translation - without write-back, or one the translation refused with - ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the - originals and the step passes, so the client gets the stream the merge base sent.""" - observer: Final = _StreamRewriteObserver(callback) + text rewrites on translations that support ended-stream write-back. A guardrail + without the unified interface runs its legacy post-call hook against the assembled + response through ``_LegacyHookStreamAdapter``. A rewrite that cannot reach the client + yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one + the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: + the buffered chunks go back to the originals and the step passes, so the client gets + the stream the merge base sent.""" + scanner: Final = ( + callback + if PipelineExecutor.supports_unified_execution(callback) + else _LegacyHookStreamAdapter(callback, endpoint_translation, user_api_key_dict) + ) + observer: Final = _StreamRewriteObserver(scanner) deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites originals: Final = copy.deepcopy(streaming_chunks) try: @@ -379,11 +460,11 @@ class PipelineExecutor: if isinstance(response, dict): callback.mark_pre_call_hook_ran(response) elif mode == "post_call" and streaming_chunks is not None: - if not use_unified or endpoint_translation is None: + if endpoint_translation is None: return ( "error", None, - f"Guardrail '{step.guardrail}' does not support streaming pipeline execution", + f"Guardrail '{step.guardrail}' cannot run on a stream without an endpoint translation", None, ) await PipelineExecutor._run_streaming_step( @@ -433,10 +514,20 @@ class PipelineExecutor: @staticmethod def supports_unified_execution(callback: CustomGuardrail) -> bool: - """Whether this guardrail runs through the unified apply_guardrail path, - the interface streaming pipeline execution requires.""" + """Whether this guardrail runs through the unified apply_guardrail path.""" return "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks + @staticmethod + def supports_streaming_execution(callback: CustomGuardrail) -> bool: + """Whether a streaming pipeline step can run this guardrail against the buffered + stream: through the unified path, or through its own post-call hook on the + assembled response. A guardrail with neither (one that only rewrites the stream + through its iterator hook) has to keep running on its own.""" + return ( + PipelineExecutor.supports_unified_execution(callback) + or type(callback).async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + ) + @staticmethod def find_guardrail_callback(guardrail_name: str) -> CustomGuardrail | None: """Look up an initialized guardrail callback by name from litellm.callbacks.""" diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8b36061c6be..bbec1b842c4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -451,7 +451,7 @@ def _pipeline_step_guardrail_names(pipelines: Sequence[tuple[str, "GuardrailPipe return frozenset(step.guardrail for _policy_name, pipeline in pipelines for step in pipeline.steps) -def _pipeline_managed_guardrail_names( +def pipeline_managed_guardrail_names( data: Mapping[str, object], mode: Literal["pre_call", "post_call"] ) -> frozenset[str]: return _pipeline_step_guardrail_names( @@ -514,9 +514,9 @@ def _merge_pipeline_metadata_writes( _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key)) -def _pipeline_step_supports_unified_streaming(guardrail_name: str) -> bool: +def _pipeline_step_supports_streaming(guardrail_name: str) -> bool: callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name) - return callback is not None and PipelineExecutor.supports_unified_execution(callback) + return callback is not None and PipelineExecutor.supports_streaming_execution(callback) def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]: @@ -541,14 +541,15 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: unsupported: Final = tuple( dict.fromkeys( - step.guardrail for step in pipeline.steps if not _pipeline_step_supports_unified_streaming(step.guardrail) + step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) ) ) if not unsupported: return True verbose_proxy_logger.warning( - "Policy '%s' has post_call pipeline guardrails without the unified apply_guardrail interface, " - "which streaming pipelines need; the stream skips the pipeline and its guardrails run on their own: %s", + "Policy '%s' has post_call pipeline guardrails with neither the unified apply_guardrail interface nor a " + "post-call hook, one of which streaming pipelines need; the stream skips the pipeline and its guardrails " + "run on their own: %s", policy_name, ", ".join(unsupported), ) @@ -562,11 +563,12 @@ def _streamable_post_call_pipelines( The post_call pipelines a streaming response can be gated through. Streaming pipelines scan the buffered stream through the endpoint guardrail - translation of the request route, so every step's guardrail needs the - unified apply_guardrail interface and the route needs a translation. A - pipeline that cannot be run that way yet is left out and its guardrails - run on the stream on their own, the way they did before pipelines ran on - streams at all, with a warning naming the pipeline. + translation of the request route, so every step's guardrail needs either the + unified apply_guardrail interface or a post-call hook to run against the + assembled response, and the route needs a translation. A pipeline that + cannot be run that way yet is left out and its guardrails run on the stream + on their own, the way they did before pipelines ran on streams at all, with + a warning naming the pipeline. """ post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: @@ -1968,7 +1970,7 @@ class ProxyLogging: ) # Get pipeline-managed guardrails to skip in normal loop - pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "pre_call") + pipeline_managed: Final = pipeline_managed_guardrail_names(data, "pre_call") caps: Final = ProxyLogging._callback_capabilities() # Skip the per-request callback walk entirely when nothing in @@ -2956,7 +2958,7 @@ class ProxyLogging: if pipeline_response is not None: response = pipeline_response # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below - pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "post_call") + pipeline_managed: Final = pipeline_managed_guardrail_names(data, "post_call") guardrail_callbacks, other_callbacks = _partition_post_call_callbacks() try: # Merge model-level guardrails before checking which guardrails to run @@ -3272,7 +3274,7 @@ class ProxyLogging: _cached_guardrail_data: dict | None = None _guardrail_data_computed = False pipeline_managed: Final = ( - _pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset() + pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset() ) for callback in litellm.callbacks: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bf40f781fa3..a10a2aae0ee 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -2156,3 +2156,29 @@ class TestAnthropicMessagesHandlerStreamingScanKey: assert open_key == StreamingScanKey(texts=("hi",)) assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] assert ended_key != open_key + + +class TestAnthropicMessagesHandlerPostCallHookResponse: + def test_openai_shaped_stream_assembly_reaches_the_hook_as_a_messages_response(self): + from litellm.types.utils import Choices, Message, ModelResponse, Usage + + assembled = ModelResponse( + id="msg_1", + model="claude", + choices=[Choices(message=Message(role="assistant", content="hello world"), finish_reason="stop")], + usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + + hook_response = AnthropicMessagesHandler().post_call_hook_response(assembled) + + assert hook_response["type"] == "message" + assert hook_response["role"] == "assistant" + assert hook_response["content"] == [{"type": "text", "text": "hello world"}] + assert hook_response["stop_reason"] == "end_turn" + assert hook_response["usage"]["input_tokens"] == 1 + assert hook_response["usage"]["output_tokens"] == 2 + + def test_anything_else_reaches_the_hook_untouched(self): + native = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]} + + assert AnthropicMessagesHandler().post_call_hook_response(native) is native diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 54ef1f79f4d..db403a17c28 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -534,7 +534,6 @@ async def test_guardrail_not_found_uses_on_fail(monkeypatch): ], ) - monkeypatch.setattr(litellm, "callbacks", []) result = await PipelineExecutor.execute_steps( steps=pipeline.steps, @@ -1153,3 +1152,152 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] + + +class _LegacyHookGuardrail(CustomGuardrail): + """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" + + def __init__(self, replacement=None, raises=None): + super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + self.replacement = replacement + self.raises = raises + self.calls = [] + + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + self.calls.append({"data": data, "user_api_key_dict": user_api_key_dict, "response": response}) + if self.raises is not None: + raise self.raises + return self.replacement + + +class _NativeHooksGuardrail(_LegacyHookGuardrail): + use_native_lifecycle_hooks = True + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise AssertionError("a guardrail that keeps its native hooks never runs apply_guardrail") + + +class _LegacyScanningTranslation: + """Stores the assembled response under request_data["response"] before scanning, like the + chat, Responses, and Messages handlers, hands hooks a route-native shape, and re-extracts one + text per entry of a replacement's "texts".""" + + delivers_ended_stream_text_rewrites = True + + def post_call_hook_response(self, response): + return {"native": True, "text": response["text"]} + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) + outputs = await guardrail_to_apply.apply_guardrail( + inputs={"texts": [responses_so_far[0]["text"]]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + responses_so_far[0]["text"] = outputs["texts"][0] + return responses_so_far + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": list(response["texts"])}, + request_data={"response": response}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return response + + +async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + return await PipelineExecutor.execute_steps( + steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail=on_fail, on_error=on_error)], + mode="post_call", + data={"model": "m"}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="p", + streaming_chunks=chunks, + endpoint_translation=_LegacyScanningTranslation(), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("guardrail_class", [_LegacyHookGuardrail, _NativeHooksGuardrail]) +async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypatch, caplog, guardrail_class): + guardrail = guardrail_class(replacement={"texts": ["[REWRITTEN] hello world"]}) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert [call["response"] for call in guardrail.calls] == [{"native": True, "text": "hello world"}] + assert guardrail.calls[0]["data"]["model"] == "m" + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_passes_untouched_when_legacy_hook_returns_none(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=None) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert len(guardrail.calls) == 1 + assert chunks == [_chunk()] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") +@pytest.mark.asyncio +async def test_streaming_step_blocks_with_the_legacy_hook_exception(monkeypatch): + exc = HTTPException(status_code=400, detail={"error": "output blocked"}) + chunks = [_chunk()] + + result = await _run_legacy_streaming_step(monkeypatch, _LegacyHookGuardrail(raises=exc), chunks) + + assert result.terminal_action == "block" + assert [step.outcome for step in result.step_results] == ["fail"] + assert result.original_exception is exc + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_takes_on_error_when_legacy_hook_crashes(monkeypatch): + chunks = [_chunk()] + + result = await _run_legacy_streaming_step( + monkeypatch, _LegacyHookGuardrail(raises=ValueError("boom")), chunks, on_error="block" + ) + + assert result.terminal_action == "block" + assert [step.outcome for step in result.step_results] == ["error"] + assert result.step_results[0].error_detail == "boom" + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement={"texts": ["split", "in two"]}) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 9f1321aec2c..f4165f98ba5 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -671,6 +671,32 @@ async def test_deferred_stream_guardrails_run_native_hook_when_opted_out(monkeyp assert routed.native_hooks_ran == [] +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monkeypatch): + """A post_call pipeline step already ran the opted-out guardrail's own hook against + the buffered stream, so the deferred audit must not run it a second time.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed]) + pipeline = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="keeps_native", on_fail="block")]) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == [] + + @pytest.mark.asyncio async def test_realtime_guardrails_skip_opted_out_guardrail(monkeypatch): """The realtime path calls apply_guardrail directly, so the opt-out has to be diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 73dd6746b29..d01f881c446 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio import json +from copy import deepcopy import logging from typing import Any, Callable, Dict, List from unittest.mock import AsyncMock, MagicMock, patch @@ -1497,29 +1498,78 @@ async def _async_chunk_iter(chunks: List[Any]): yield chunk -def test_streamable_post_call_pipelines_keeps_supported_and_drops_unsupported( +def _legacy_hook_stream_guardrail( + seen: Dict[str, Any], + rewrite: Callable[[Any], Any] | None = None, + raises: Exception | None = None, + native_lifecycle: bool = False, +) -> CustomGuardrail: + class LegacyHookGuardrail(CustomGuardrail): + use_native_lifecycle_hooks = native_lifecycle + + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + seen["count"] = seen.get("count", 0) + 1 + seen["data"] = data + seen["user_api_key_dict"] = user_api_key_dict + seen["response"] = deepcopy(response) + if raises is not None: + raise raises + return None if rewrite is None else rewrite(response) + + if native_lifecycle: + + class NativeLifecycleGuardrail(LegacyHookGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise AssertionError("a guardrail that keeps its native hooks never runs apply_guardrail") + + return NativeLifecycleGuardrail( + guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False + ) + return LegacyHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False) + + +def _iterator_hook_only_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuardrail: + class IteratorHookGuardrail(CustomGuardrail): + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + seen["count"] = seen.get("count", 0) + 1 + async for item in response: + item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" + yield item + + return IteratorHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) + + +def _rewritten_model_response(response: Any) -> litellm.ModelResponse: + payload = response.model_dump() + payload["choices"][0]["message"]["content"] = "[REWRITTEN] " + payload["choices"][0]["message"]["content"] + return litellm.ModelResponse(**payload) + + +def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator_only( make_user_api_key_auth, monkeypatch, caplog ): - class NativeOnlyGuardrail(CustomGuardrail): - pass - supported = _unified_stream_guardrail({}) - native_only = NativeOnlyGuardrail(guardrail_name="gr-native", event_hook=GuardrailEventHooks.post_call) - monkeypatch.setattr(litellm, "callbacks", [supported, native_only]) - governed = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + legacy = _legacy_hook_stream_guardrail({}) + legacy.guardrail_name = "gr-legacy" + iterator_only = _iterator_hook_only_guardrail("gr-iterator", {}) + monkeypatch.setattr(litellm, "callbacks", [supported, legacy, iterator_only]) + governed = GuardrailPipeline( + mode="post_call", + steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-legacy", on_fail="block")], + ) ungoverned = GuardrailPipeline( mode="post_call", - steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-native", on_fail="block")], + steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-iterator", on_fail="block")], ) - pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-native", on_fail="block")]) + pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-iterator", on_fail="block")]) data = {"metadata": {"_guardrail_pipelines": [("governed", governed), ("ungoverned", ungoverned), ("req", pre_call)]}} with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): streamable = _streamable_post_call_pipelines(data, make_user_api_key_auth(request_route="/v1/chat/completions")) assert streamable == (("governed", governed),) - assert any("'ungoverned'" in message and "gr-native" in message for message in _warnings(caplog)) - assert not any("'governed'" in message for message in _warnings(caplog)) + assert any("'ungoverned'" in message and "gr-iterator" in message for message in _warnings(caplog)) + assert not any("'governed'" in message or "gr-legacy" in message for message in _warnings(caplog)) def test_streamable_post_call_pipelines_is_empty_on_route_without_translation( @@ -1569,55 +1619,123 @@ async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_u @pytest.mark.asyncio @pytest.mark.parametrize("native_lifecycle", [False, True]) -async def test_streaming_iterator_hook_releases_stream_when_pipeline_guardrail_lacks_unified_support( +async def test_streaming_iterator_hook_runs_legacy_hook_and_delivers_its_rewrite( proxy_logging, make_user_api_key_auth, monkeypatch, native_lifecycle, caplog ): seen: Dict[str, Any] = {} - if native_lifecycle: - - class NativeOnlyGuardrail(CustomGuardrail): - use_native_lifecycle_hooks = True - - async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): - seen["count"] = seen.get("count", 0) + 1 - return inputs - - else: - - class NativeOnlyGuardrail(CustomGuardrail): - async def async_post_call_success_hook(self, data, user_api_key_dict, response): - seen["count"] = seen.get("count", 0) + 1 - return response - - monkeypatch.setattr( - litellm, - "callbacks", - [NativeOnlyGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)], - ) + guardrail = _legacy_hook_stream_guardrail(seen, rewrite=_rewritten_model_response, native_lifecycle=native_lifecycle) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) chunks = _stream_chunks() - delivered: List[Any] = [] + auth = make_user_api_key_auth(request_route="/v1/chat/completions") with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): out = await proxy_logging.pre_call_hook( - user_api_key_dict=make_user_api_key_auth(), - data=data, - call_type="completion", - guardrails_only=True, + user_api_key_dict=auth, data=data, call_type="completion", guardrails_only=True ) + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data + ) + ] + + assert out is not None and out.get("stream") is True + assert seen["count"] == 1 + assert isinstance(seen["response"], litellm.ModelResponse) + assert seen["response"].choices[0].message.content == "hello world" + assert seen["data"]["messages"] == data["messages"] + assert seen["user_api_key_dict"] is auth + assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks] + assert delivered[0].choices[0].delta.content == "[REWRITTEN] hello world" + assert delivered[1].choices[0].delta.content in (None, "") + assert delivered[1].choices[0].finish_reason == "stop" + assert data["metadata"]["applied_guardrails"] == ["gr-post"] + assert _warnings(caplog) == [] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_releases_stream_untouched_when_legacy_hook_returns_none( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + chunks = _stream_chunks() + + delivered = [ + item async for item in proxy_logging.async_post_call_streaming_iterator_hook( user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), response=_async_chunk_iter(chunks), request_data=data, + ) + ] + + assert seen["count"] == 1 + assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks] + assert [item.choices[0].delta.content for item in delivered] == ["hello ", "world"] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_ends_stream_with_legacy_hook_exception( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + blocked = HTTPException(status_code=400, detail={"error": "output blocked"}) + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, raises=blocked)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + delivered: List[Any] = [] + + async def _drain() -> None: + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_stream_chunks()), + request_data=data, ): delivered.append(item) - assert out is not None - assert out.get("stream") is True - assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True] - assert len(delivered) == 2 - assert seen.get("count") is None - assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + with pytest.raises(HTTPException) as info: + await _drain() + + assert seen["count"] == 1 + assert delivered == [] + assert info.value is blocked + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + + def rewrite(response: Any) -> Dict[str, Any]: + return {**response, "content": [{"type": "text", "text": "[REWRITTEN] " + response["content"][0]["text"]}]} + + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, rewrite=rewrite)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_sse_chunks()), + request_data=data, + ) + ] + + assert seen["count"] == 1 + assert seen["response"]["content"][0]["text"] == "hello world" + assert seen["response"]["role"] == "assistant" + raw = b"".join(delivered).decode() + assert "[REWRITTEN] hello world" in raw + assert raw.count("event: content_block_delta") == 1 + for expected_event in ("message_start", "content_block_start", "content_block_stop", "message_delta", "message_stop"): + assert f"event: {expected_event}" in raw @pytest.mark.asyncio @@ -1625,19 +1743,7 @@ async def test_streaming_iterator_hook_runs_iterator_hook_guardrail_whose_pipeli proxy_logging, make_user_api_key_auth, monkeypatch, caplog ): seen: Dict[str, Any] = {} - - class IteratorHookGuardrail(CustomGuardrail): - async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): - seen["count"] = seen.get("count", 0) + 1 - async for item in response: - item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" - yield item - - monkeypatch.setattr( - litellm, - "callbacks", - [IteratorHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)], - ) + monkeypatch.setattr(litellm, "callbacks", [_iterator_hook_only_guardrail("gr-post", seen)]) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) From d497434680186367ce42795ae1fca314503cd2aa Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 19:28:04 +0000 Subject: [PATCH 17/81] fix(model_prices): update provider and web search pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 140 +++++++++++++++--- model_prices_and_context_window.json | 140 +++++++++++++++--- .../test_tool_call_cost_tracking.py | 65 +++++++- ...tex_ai_gemini_transcribe_transformation.py | 4 +- 4 files changed, 293 insertions(+), 56 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef8da798922..a540df1c7d1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29176,7 +29176,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, + "supports_web_search": true }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29193,9 +29199,9 @@ "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29204,7 +29210,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", @@ -29294,9 +29301,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29321,9 +29328,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29434,9 +29441,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29461,9 +29468,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -41625,6 +41632,28 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "rerank-v4.0-fast": { + "input_cost_per_query": 0.002, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, + "rerank-v4.0-pro": { + "input_cost_per_query": 0.0025, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -47811,7 +47840,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47827,7 +47856,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47842,14 +47871,17 @@ }, "vertex_ai/xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -47858,14 +47890,17 @@ }, "vertex_ai/xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47873,6 +47908,42 @@ "supports_vision": true, "supports_web_search": true }, + "vertex_ai/xai/grok-4.3": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vertex_ai/xai/grok-4.6": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, "litellm_provider": "vertex_ai-qwen_models", @@ -56824,8 +56895,8 @@ "rpm": 10 }, "vertex_ai/gemini-3.5-transcribe-preview": { - "input_cost_per_audio_token": 2.5e-06, - "input_cost_per_token": 2.5e-06, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, @@ -56860,6 +56931,27 @@ ], "supports_audio_input": true }, + "vertex_ai/gemini-3.5-live-translate-preview": { + "input_cost_per_audio_token": 3.5e-06, + "input_cost_per_token": 3.5e-06, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_token": 2.1e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef8da798922..a540df1c7d1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29176,7 +29176,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, + "supports_web_search": true }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29193,9 +29199,9 @@ "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29204,7 +29210,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", @@ -29294,9 +29301,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29321,9 +29328,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29434,9 +29441,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29461,9 +29468,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -41625,6 +41632,28 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "rerank-v4.0-fast": { + "input_cost_per_query": 0.002, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, + "rerank-v4.0-pro": { + "input_cost_per_query": 0.0025, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -47811,7 +47840,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47827,7 +47856,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47842,14 +47871,17 @@ }, "vertex_ai/xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -47858,14 +47890,17 @@ }, "vertex_ai/xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47873,6 +47908,42 @@ "supports_vision": true, "supports_web_search": true }, + "vertex_ai/xai/grok-4.3": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vertex_ai/xai/grok-4.6": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, "litellm_provider": "vertex_ai-qwen_models", @@ -56824,8 +56895,8 @@ "rpm": 10 }, "vertex_ai/gemini-3.5-transcribe-preview": { - "input_cost_per_audio_token": 2.5e-06, - "input_cost_per_token": 2.5e-06, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, @@ -56860,6 +56931,27 @@ ], "supports_audio_input": true }, + "vertex_ai/gemini-3.5-live-translate-preview": { + "input_cost_per_audio_token": 3.5e-06, + "input_cost_per_token": 3.5e-06, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_token": 2.1e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 6cc3dcceebc..38bf8c544c7 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,5 +1,6 @@ -import os +import json from collections.abc import Mapping, Sequence +from pathlib import Path import pytest @@ -11,8 +12,6 @@ from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebS from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams - - def test_web_search_cost_low(): web_search_options = WebSearchOptions(search_context_size="low") model_info = litellm.get_model_info("gpt-4o-search-preview") @@ -683,12 +682,13 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( def _openai_responses_with_web_search_calls(model, num_calls): - from litellm.types.llms.openai import ResponsesAPIResponse from openai.types.responses.response_function_web_search import ( ActionSearch, ResponseFunctionWebSearch, ) + from litellm.types.llms.openai import ResponsesAPIResponse + output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -859,11 +859,64 @@ def test_dated_search_preview_entries_carry_search_pricing(local_model_cost_map) custom_llm_provider="openai", standard_built_in_tools_params=None, ) - assert cost == pytest.approx(0.035), ( - f"dated search-preview id must bill the $0.035 search fee, got ${cost}" + assert cost == pytest.approx(0.025), ( + f"dated search-preview id must bill the $0.025 search fee, got ${cost}" ) +@pytest.mark.parametrize( + "web_search_options", + [ + None, + WebSearchOptions(search_context_size="low"), + WebSearchOptions(search_context_size="medium"), + WebSearchOptions(search_context_size="high"), + ], +) +def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( + web_search_options, local_model_cost_map +): + """Snapshot and alias must bill web search identically: OpenAI lists preview search at $25 per 1k calls.""" + alias_info = litellm.get_model_info("gpt-4o-mini") + snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") + + assert snapshot_info["supports_web_search"] is True + assert alias_info["supports_web_search"] is True + + snapshot_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=web_search_options, model_info=snapshot_info + ) + alias_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=web_search_options, model_info=alias_info + ) + + assert snapshot_cost == alias_cost == 0.025 + + +def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps(): + """The bundled backup and canonical file are both served to deployments, so both must carry the price.""" + repo_root = Path(__file__).parents[4] + cost_maps = tuple( + json.loads((repo_root / path).read_text(encoding="utf-8")) + for path in ( + "model_prices_and_context_window.json", + "litellm/model_prices_and_context_window_backup.json", + ) + ) + canonical, backup = cost_maps + expected_search_price = { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025, + "search_context_size_high": 0.025, + } + for model_name in ("gpt-4o-mini", "gpt-4o-mini-2024-07-18"): + canonical_entry = canonical[model_name] + backup_entry = backup[model_name] + assert canonical_entry["search_context_cost_per_query"] == expected_search_price + assert backup_entry["search_context_cost_per_query"] == expected_search_price + assert canonical_entry == backup_entry + + # Note: File search integration test removed due to complex annotation detection logic # The unit tests in test_azure_assistant_cost_tracking.py provide comprehensive coverage diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index eadd87d9c92..08e46b1ffac 100644 --- a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -322,8 +322,8 @@ class TestModelCostEntry: entry = json.load(f)["vertex_ai/gemini-3.5-transcribe-preview"] assert entry["mode"] == "audio_transcription" assert entry["litellm_provider"] == "vertex_ai" - assert entry["input_cost_per_audio_token"] == pytest.approx(2.5e-06) - assert entry["input_cost_per_token"] == pytest.approx(2.5e-06) + assert entry["input_cost_per_audio_token"] == pytest.approx(2e-06) + assert entry["input_cost_per_token"] == pytest.approx(2e-06) assert entry["output_cost_per_token"] == pytest.approx(1.2e-05) assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"] From 3f30c05493091f62d1edcfb0af58aa33a370a335 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:39:39 -0700 Subject: [PATCH 18/81] fix(guardrails): skip only stream-gated pipeline guardrails in the deferred post-call pass --- litellm/proxy/common_request_processing.py | 6 +- litellm/proxy/utils.py | 37 +++++++++-- .../test_proxy_logging_hook_detection.py | 63 +++++++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7e85b66682c..2a6eb0a16ff 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3343,15 +3343,15 @@ class ProxyBaseLLMRequestProcessing: from litellm.proxy.proxy_server import llm_router as _global_llm_router from litellm.proxy.utils import ( _check_and_merge_model_level_guardrails, - pipeline_managed_guardrail_names, + stream_gated_guardrail_names, ) guardrail_data = _check_and_merge_model_level_guardrails(data=captured_data, llm_router=_global_llm_router) - pipeline_managed: Final = pipeline_managed_guardrail_names(captured_data, "post_call") + stream_gated: Final = stream_gated_guardrail_names(captured_data, captured_user_api_key_dict) for cb in litellm.callbacks: if not isinstance(cb, CustomGuardrail): continue - if cb.guardrail_name in pipeline_managed: + if cb.guardrail_name in stream_gated: continue if not cb.should_run_guardrail( data=guardrail_data, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bbec1b842c4..106fab7af8a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -538,12 +538,16 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No ) -def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: - unsupported: Final = tuple( +def _pipeline_unsupported_streaming_guardrails(pipeline: "GuardrailPipeline") -> tuple[str, ...]: + return tuple( dict.fromkeys( step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) ) ) + + +def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: + unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline) if not unsupported: return True verbose_proxy_logger.warning( @@ -573,13 +577,12 @@ def _streamable_post_call_pipelines( post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: return () - route: Final = user_api_key_dict.request_route - if route and resolve_endpoint_translation(user_api_key_dict, None) is None: + if not _route_has_endpoint_translation(user_api_key_dict): verbose_proxy_logger.warning( "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet " "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run " "on their own: %s", - route, + user_api_key_dict.request_route, ", ".join(policy_name for policy_name, _pipeline in post_call_pipelines), ) return () @@ -590,6 +593,30 @@ def _streamable_post_call_pipelines( ) +def _route_has_endpoint_translation(user_api_key_dict: UserAPIKeyAuth) -> bool: + return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None + + +def stream_gated_guardrail_names( + request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth +) -> frozenset[str]: + """ + The guardrails whose post_call pipelines gate a streaming response on this + route: the selection ``_streamable_post_call_pipelines`` makes, without its + warnings, so the post-call pass deferred to the end of the stream skips + exactly the guardrails the pipelines already ran and no others. + """ + if not _route_has_endpoint_translation(user_api_key_dict): + return frozenset() + return _pipeline_step_guardrail_names( + tuple( + (policy_name, pipeline) + for policy_name, pipeline in _post_call_pipelines(request_data) + if not _pipeline_unsupported_streaming_guardrails(pipeline) + ) + ) + + def _prompt_block_text(block: object) -> str: if isinstance(block, str): return block diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index f4165f98ba5..10de7e7fc3e 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -697,6 +697,69 @@ async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monk assert pipeline_managed.native_hooks_ran == [] +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_run_native_hook_whose_pipeline_could_not_stream(monkeypatch): + """A pipeline step with neither streaming interface keeps the whole pipeline off the + stream, so the deferred audit is the only place the opted-out guardrail's own hook + still runs, the way it did before pipelines ran on streams.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + class NeitherHookGuardrail(CustomGuardrail): + pass + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + neither = NeitherHookGuardrail(guardrail_name="gr-neither", event_hook=GuardrailEventHooks.post_call) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed, neither]) + pipeline = GuardrailPipeline( + mode="post_call", + steps=[ + PipelineStep(guardrail="keeps_native", on_fail="next"), + PipelineStep(guardrail="gr-neither", on_fail="block"), + ], + ) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == ["post_call"] + + +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_run_native_hook_on_route_without_translation(monkeypatch): + """A route with no endpoint guardrail translation cannot gate the stream through its + pipelines, so the deferred audit still owes the opted-out guardrail its own hook.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed]) + pipeline = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="keeps_native", on_fail="block")]) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/custom/stream"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == ["post_call"] + + @pytest.mark.asyncio async def test_realtime_guardrails_skip_opted_out_guardrail(monkeypatch): """The realtime path calls apply_guardrail directly, so the opt-out has to be From 8eb6f6d8350dd0ef7f2e5a61b042213e0b6bd319 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 19:43:32 +0000 Subject: [PATCH 19/81] test(cost_calc): type web search test params and drop docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/test_tool_call_cost_tracking.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 38bf8c544c7..f319413045f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -874,9 +874,8 @@ def test_dated_search_preview_entries_carry_search_pricing(local_model_cost_map) ], ) def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( - web_search_options, local_model_cost_map -): - """Snapshot and alias must bill web search identically: OpenAI lists preview search at $25 per 1k calls.""" + web_search_options: WebSearchOptions | None, local_model_cost_map: None +) -> None: alias_info = litellm.get_model_info("gpt-4o-mini") snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") @@ -894,7 +893,6 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps(): - """The bundled backup and canonical file are both served to deployments, so both must carry the price.""" repo_root = Path(__file__).parents[4] cost_maps = tuple( json.loads((repo_root / path).read_text(encoding="utf-8")) From f59354d09b642bf50d888a82b5fac5b96d34169b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:22:29 -0700 Subject: [PATCH 20/81] refactor(guardrails): patch Anthropic SSE rewrites by field The ended-stream rewriters now describe the one field they change as a frozen _SSEFieldRewrite and a single applier builds the patched event, so the handler adds no mutable-collection constructions over staging's total --- .../chat/guardrail_translation/handler.py | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2049866b444..cbbccac17f3 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,7 +13,7 @@ Pattern Overview: """ import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat @@ -161,7 +161,25 @@ class _ToolCallShape: arguments: str -_SSEEventRewriter = Callable[[Mapping[str, object]], Mapping[str, object] | None] +@dataclass(frozen=True, slots=True) +class _SSEFieldRewrite: + """One field of one nested section of a buffered SSE event, rewritten.""" + + section: str + field: str + value: object + + +class _SSEEventRewriter(Protocol): + def __call__(self, event: Mapping[str, object]) -> _SSEFieldRewrite | None: ... + + +def _rewritten_event(event: Mapping[str, object], rewrite_event: _SSEEventRewriter) -> Mapping[str, object]: + rewrite: Final = rewrite_event(event) + section: Final = None if rewrite is None else event.get(rewrite.section) + if rewrite is None or not isinstance(section, Mapping): + return event + return {**event, rewrite.section: {**section, rewrite.field: rewrite.value}} # mutable-ok: json.dumps needs a dict def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]: @@ -1253,13 +1271,13 @@ class AnthropicMessagesHandler(BaseTranslation): message and content-block framing untouched.""" replacements: Final = chain((rewritten_text,), repeat("")) - def rewrite_text_delta(event: Mapping[str, object]) -> Mapping[str, object] | None: + def rewrite_text_delta(event: Mapping[str, object]) -> _SSEFieldRewrite | None: delta: Final = event.get("delta") if event.get("type") != "content_block_delta" or not isinstance(delta, Mapping): return None if delta.get("type") != "text_delta": return None - return {**event, "delta": {**delta, "text": next(replacements)}} + return _SSEFieldRewrite("delta", "text", next(replacements)) AnthropicMessagesHandler._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta) @@ -1306,22 +1324,21 @@ class AnthropicMessagesHandler(BaseTranslation): {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_block.items()} ) - def rewrite_tool_use(event: Mapping[str, object]) -> Mapping[str, object] | None: + def rewrite_tool_use(event: Mapping[str, object]) -> _SSEFieldRewrite | None: index: Final = event.get("index") if not isinstance(index, int) or index not in rewrites_by_block: return None match event.get("type"): case "content_block_start": - block: Final = event.get("content_block") name: Final = rewrites_by_block[index].name - if not isinstance(block, Mapping) or name is None: + if name is None: return None - return {**event, "content_block": {**block, "name": name}} + return _SSEFieldRewrite("content_block", "name", name) case "content_block_delta": delta: Final = event.get("delta") if not isinstance(delta, Mapping) or delta.get("type") != "input_json_delta": return None - return {**event, "delta": {**delta, "partial_json": next(argument_replacements[index])}} + return _SSEFieldRewrite("delta", "partial_json", next(argument_replacements[index])) case _: return None @@ -1343,8 +1360,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: if isinstance(item, dict): - rewritten: Final = rewrite_event(_as_str_mapping(item)) - return item if rewritten is None else dict(rewritten) + return _rewritten_event(_as_str_mapping(item), rewrite_event) if isinstance(item, (bytes, bytearray)): return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) return item @@ -1374,8 +1390,8 @@ class AnthropicMessagesHandler(BaseTranslation): return line if not isinstance(data, dict): return line - rewritten: Final = rewrite_event(_as_str_mapping(data)) - return line if rewritten is None else "data: " + json.dumps(rewritten) + rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event) + return line if rewritten is data else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) From 89e11949c834d0b466375b0442b27304946568a9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:22:30 -0700 Subject: [PATCH 21/81] fix(guardrails): key Responses stream tool-call rewrites by call_id Bridged Responses streams give reasoning and message items output_index 0 and start function calls at 1, so keying rewrites by output_index rewrote the wrong items. Rewrites now follow each function call's call_id through the buffered item and argument events, refuse when an event cannot be resolved to a rewritten call, and the refusal branches on all three handlers get regression tests --- .../guardrail_translation/handler.py | 132 ++++++++++++------ .../test_anthropic_guardrail_handler.py | 25 ++++ .../test_openai_guardrail_handler.py | 56 ++++++++ ...test_openai_responses_guardrail_handler.py | 90 ++++++++++++ 4 files changed, 259 insertions(+), 44 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 208ceb959e6..280a8670c36 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -140,6 +140,10 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( ) +_FUNCTION_CALL_ARGUMENT_EVENT_TYPES: Final = frozenset( + {"response.function_call_arguments.delta", "response.function_call_arguments.done"} +) +_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "message": "content"} ) @@ -930,70 +934,110 @@ class OpenAIResponsesHandler(BaseTranslation): ) -> None: """Write ended-stream guardrail tool-call rewrites into the completed envelope's ``function_call`` items and sync the earlier stream events, - keyed by ``output_index``. The guardrail sees the envelope's function - calls in output order, which is how a rewritten call finds its item; a - rewrite whose calls do not line up with the envelope is reported as - undeliverable, so the pipeline executor discards it and releases the - original events.""" + keyed by ``call_id``. The guardrail sees the envelope's function calls + in output order, which is how a rewritten call finds its ``call_id``; + the stream events find their call through the ``call_id`` on + ``output_item`` events and the ``item_id`` on argument events, since an + event's ``output_index`` need not match the envelope's (the chat bridge + numbers tool calls from 1 while the envelope lists them after the + message). A rewrite whose calls do not line up with the envelope, or + whose events cannot be found, is reported as undeliverable, so the + pipeline executor discards it and releases the original events.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - function_call_indices: Final = tuple( - output_idx - for output_idx, output_item in enumerate(outputs) - if stream_item_field(output_item, "type") == "function_call" + function_call_items: Final = tuple( + output_item for output_item in outputs if stream_item_field(output_item, "type") == "function_call" ) - if len(function_call_indices) != len(post_guardrail_tool_calls): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite(guardrail_name) - rewrites_by_output_index: Final = MappingProxyType( + call_ids: Final = tuple( + call_id + for output_item in function_call_items + if isinstance(call_id := stream_item_field(output_item, "call_id"), str) and call_id + ) + stream_events: Final = responses_so_far[:-1] + call_id_by_item_id: Final = self._function_call_ids_by_item_id(stream_events) + event_call_ids: Final = tuple( + self._function_call_event_call_id(event, call_id_by_item_id) for event in stream_events + ) + rewrites_by_call_id: Final = MappingProxyType( { - output_idx: after - for output_idx, before, after in zip( - function_call_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls - ) + call_id: after + for call_id, before, after in zip(call_ids, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before } ) - for output_idx, rewrite in rewrites_by_output_index.items(): - self._write_function_call_item(outputs[output_idx], rewrite.name, rewrite.arguments) - self._sync_stream_events_with_tool_call_rewrites( - stream_events=responses_so_far[:-1], - rewrites_by_output_index=rewrites_by_output_index, + unresolved_argument_event: Final = any( + call_id is None and stream_item_field(event, "type") in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES + for event, call_id in zip(stream_events, event_call_ids) ) + if ( + len(call_ids) != len(function_call_items) + or len(frozenset(call_ids)) != len(call_ids) + or len(call_ids) != len(post_guardrail_tool_calls) + or unresolved_argument_event + or not rewrites_by_call_id.keys() <= frozenset(event_call_ids) + ): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - def _sync_stream_events_with_tool_call_rewrites( - self, - stream_events: Sequence[object], - rewrites_by_output_index: Mapping[int, _ToolCallShape], - ) -> None: - """Sync pre-completion function-call events with the rewritten completed - response: the first ``function_call_arguments.delta`` for a rewritten call - carries the full rewritten arguments and the rest are blanked, while - ``function_call_arguments.done`` and ``output_item.done`` carry the full - rewritten arguments and ``output_item.added`` / ``output_item.done`` the - rewritten name, so every event a client may read agrees with the - rewritten ``response.completed`` payload.""" + raise UndeliverableStreamRewrite(guardrail_name) + for output_item, rewrite in ( + (output_item, rewrites_by_call_id[call_id]) + for output_item, call_id in zip(function_call_items, call_ids) + if call_id in rewrites_by_call_id + ): + self._write_function_call_item(output_item, rewrite.name, rewrite.arguments) delta_replacements: Final = MappingProxyType( - {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_output_index.items()} + {call_id: chain((rewrite.arguments,), repeat("")) for call_id, rewrite in rewrites_by_call_id.items()} ) - for event in stream_events: - output_index = stream_item_field(event, "output_index") - if not isinstance(output_index, int) or output_index not in rewrites_by_output_index: + for event, call_id in zip(stream_events, event_call_ids): + if call_id not in rewrites_by_call_id: continue - rewrite = rewrites_by_output_index[output_index] match stream_item_field(event, "type"): case "response.function_call_arguments.delta": - self._write_event_field(event, "delta", next(delta_replacements[output_index])) + self._write_event_field(event, "delta", next(delta_replacements[call_id])) case "response.function_call_arguments.done": - self._write_event_field(event, "arguments", rewrite.arguments) + self._write_event_field(event, "arguments", rewrites_by_call_id[call_id].arguments) case "response.output_item.added": - self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, None) + self._write_function_call_item( + stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, None + ) case "response.output_item.done": - self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, rewrite.arguments) + self._write_function_call_item( + stream_item_field(event, "item"), + rewrites_by_call_id[call_id].name, + rewrites_by_call_id[call_id].arguments, + ) case _: pass + @staticmethod + def _function_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: + items: Final = tuple( + stream_item_field(event, "item") + for event in stream_events + if stream_item_field(event, "type") in _OUTPUT_ITEM_EVENT_TYPES + ) + return MappingProxyType( + { + item_id: call_id + for item in items + if stream_item_field(item, "type") == "function_call" + and isinstance(item_id := stream_item_field(item, "id"), str) + and isinstance(call_id := stream_item_field(item, "call_id"), str) + } + ) + + @staticmethod + def _function_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: + event_type: Final = stream_item_field(event, "type") + if event_type in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES: + item_id: Final = stream_item_field(event, "item_id") + return call_id_by_item_id.get(item_id) if isinstance(item_id, str) else None + if event_type not in _OUTPUT_ITEM_EVENT_TYPES: + return None + item: Final = stream_item_field(event, "item") + call_id: Final = stream_item_field(item, "call_id") + return call_id if stream_item_field(item, "type") == "function_call" and isinstance(call_id, str) else None + @staticmethod def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: if item is None: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bd2dff9b33c..64c243343d5 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -381,6 +381,31 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert chunks == original + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_use_rewrite_with_server_tool_use_block_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = AnthropicMessagesHandler() + server_tool_use = [ + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ] + tool_use = self._ended_tool_use_sse_chunks() + chunks = ( + tool_use[:1] + + [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in server_tool_use] + + [chunk.replace(b'"index": 0', b'"index": 1') for chunk in tool_use[1:]] + ) + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index be168ec83e6..c7011a6f00e 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1252,6 +1252,62 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: deliver_ended_stream_rewrites=True, ) + @staticmethod + def _two_choice_tool_call_stream_chunks() -> list: + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + def chunk( + choice_index: int, tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None + ) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=choice_index, + delta=Delta(tool_calls=[tool_call] if tool_call else None), + finish_reason=finish_reason, + ) + ], + ) + + def fragment(arguments: str, name: Optional[str] = None, call_id: Optional[str] = None): + return ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments) + ) + + return [ + chunk(0, fragment("", name="lookup_fruit", call_id="call_1")), + chunk(1, fragment("", name="lookup_fruit", call_id="call_2")), + chunk(0, fragment('{"fruit": "persimmon"}')), + chunk(1, fragment('{"fruit": "durian"}')), + chunk(0, None, finish_reason="tool_calls"), + chunk(1, None, finish_reason="tool_calls"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIChatCompletionsHandler() + chunks = self._two_choice_tool_call_stream_chunks() + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index bdf7f83757c..fa1e969b277 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1314,6 +1314,96 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' assert completed_event.response.output[0].name == "lookup_fruit" + @staticmethod + def _bridged_function_call_stream_events() -> List[dict]: + reasoning = {"type": "reasoning", "id": "rs_1", "summary": []} + text = {"type": "output_text", "text": "Looking that up", "annotations": []} + message = {"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [text]} + + def function_call(arguments: str, status: str) -> dict: + return { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": dict(reasoning)}, + {"type": "response.output_item.done", "output_index": 0, "item": dict(reasoning)}, + {"type": "response.output_item.added", "output_index": 0, "item": {**message, "status": "in_progress", "content": []}}, + {"type": "response.output_text.delta", "item_id": "msg_1", "output_index": 0, "content_index": 0, "delta": "Looking that up"}, + {"type": "response.output_item.done", "output_index": 0, "item": {**message, "content": [dict(text)]}}, + {"type": "response.output_item.added", "output_index": 1, "item": function_call("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 1, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 1, "delta": ' "persimmon"}'}, + { + "type": "response.function_call_arguments.done", + "item_id": "fc_1", + "output_index": 1, + "arguments": '{"fruit": "persimmon"}', + }, + {"type": "response.output_item.done", "output_index": 1, "item": function_call('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_1", + "model": "claude-haiku-4-5", + "output": [ + dict(reasoning), + {**message, "content": [dict(text)]}, + function_call('{"fruit": "persimmon"}', "completed"), + ], + }, + }, + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_keys_bridged_function_call_events_by_call_id(self): + handler = OpenAIResponsesHandler() + events = self._bridged_function_call_stream_events() + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert events[6]["delta"] == '{"fruit": "[MASKED]"}' + assert events[7]["delta"] == "" + assert events[8]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["item"]["name"] == "lookup_fruit" + assert events[9]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[10]["response"]["output"][2]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[3]["delta"] == "Looking that up" + assert events[4]["item"]["content"][0]["text"] == "Looking that up" + assert events[10]["response"]["output"][1]["content"][0]["text"] == "Looking that up" + assert events[1]["item"] == {"type": "reasoning", "id": "rs_1", "summary": []} + + @pytest.mark.asyncio + @pytest.mark.parametrize("mismatch", ["orphan_call_id", "duplicate_call_id"]) + async def test_deliver_ended_stream_function_call_rewrite_without_matching_events_fails_closed(self, mismatch): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + envelope_item = events[5]["response"]["output"][0] + if mismatch == "orphan_call_id": + events[5]["response"]["output"] = [{**envelope_item, "call_id": "call_999"}] + else: + events[5]["response"]["output"] = [dict(envelope_item), dict(envelope_item)] + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): handler = OpenAIResponsesHandler() From c1132d045470853a8a186d730678493f3f015dbe Mon Sep 17 00:00:00 2001 From: shotsan Date: Tue, 8 Sep 2026 13:44:50 -0700 Subject: [PATCH 22/81] fix(convert_dict_to_response): handle empty choices list without raising 500 APIError (Fixes #40276) --- .../convert_dict_to_response.py | 6 +- .../test_convert_dict_to_response.py | 70 ++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index a6f10e1ede3..7d8072ba622 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -179,7 +179,7 @@ async def convert_to_streaming_response_async( choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( @@ -287,7 +287,7 @@ def convert_to_streaming_response( model_response_object: Final = ModelResponseStream() choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( @@ -623,7 +623,7 @@ def convert_to_model_response_object( return convert_to_streaming_response(response_object=response_object) choice_list: Final[list[Choices]] = [] - if not response_object.get("choices") or not isinstance(response_object["choices"], Iterable): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 304d732c518..5406df691aa 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,4 +1,4 @@ - +import pytest from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -99,3 +99,71 @@ def test_handle_invalid_parallel_tool_calls_skips_custom_tool_calls(): ) result = _handle_invalid_parallel_tool_calls([custom_tool_call, function_tool_call]) assert result == [custom_tool_call, function_tool_call] + + +def test_convert_empty_choices_response() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + ) + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + "vertex_ai_safety_results": ["blocked"], + } + result = convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert result.choices == [] + assert getattr(result, "vertex_ai_safety_results") == ["blocked"] + + # Test sync streaming generator handles empty choices + sync_stream = list(convert_to_streaming_response(response_object=resp)) + assert len(sync_stream) == 1 + assert sync_stream[0].choices == [] + + +@pytest.mark.asyncio +async def test_convert_empty_choices_response_async() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response_async, + ) + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + } + async_chunks = [] + async for chunk in convert_to_streaming_response_async(response_object=resp): + async_chunks.append(chunk) + assert len(async_chunks) == 1 + assert async_chunks[0].choices == [] + + +def test_convert_missing_choices_raises_api_error() -> None: + from litellm.exceptions import APIError + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + } + with pytest.raises(APIError) as exc_info: + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert "no 'choices'" in str(exc_info.value) + From e700e79cce9dac5d0fda0b5d31210283230c9de7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:52:38 -0700 Subject: [PATCH 23/81] refactor(guardrails): keep the rescanned inputs instead of rebuilding the texts list --- litellm/proxy/policy_engine/pipeline_executor.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index afc9a29afd9..089f0683823 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -128,7 +128,7 @@ class _StreamRewriteObserver(CustomGuardrail): class _ScannedTextRecorder(CustomGuardrail): def __init__(self, guardrail_name: str) -> None: super().__init__(guardrail_name=guardrail_name) - self.texts: tuple[str, ...] | None = None + self.inputs: GenericGuardrailAPIInputs | None = None @_logged_by_inner_guardrail async def apply_guardrail( @@ -138,7 +138,7 @@ class _ScannedTextRecorder(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: - self.texts = _text_snapshot(inputs.get("texts")) + self.inputs = inputs return inputs @@ -183,12 +183,16 @@ class _LegacyHookStreamAdapter(CustomGuardrail): if replacement is None: return inputs scanned: Final = _text_snapshot(inputs.get("texts")) - rewritten: Final = await self._scanned_texts(replacement, logging_obj) + rescanned: Final = await self._rescan(replacement, logging_obj) + rewritten: Final = None if rescanned is None else rescanned.get("texts") if scanned is None or rewritten is None or len(rewritten) != len(scanned): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") - return {**inputs, "texts": list(rewritten)} + rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} + return rewritten_inputs - async def _scanned_texts(self, response: object, logging_obj: "LiteLLMLoggingObj | None") -> tuple[str, ...] | None: + async def _rescan( + self, response: object, logging_obj: "LiteLLMLoggingObj | None" + ) -> GenericGuardrailAPIInputs | None: recorder: Final = _ScannedTextRecorder(self.guardrail_name or "unknown") await self.endpoint_translation.process_output_response( response=response, @@ -196,7 +200,7 @@ class _LegacyHookStreamAdapter(CustomGuardrail): litellm_logging_obj=logging_obj, user_api_key_dict=self.user_api_key_dict, ) - return recorder.texts + return recorder.inputs def _prepare_hook_input( From 80782d478ddbdd2e145b0cb72eb3c653efbe0ed0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:17:36 -0700 Subject: [PATCH 24/81] fix(policy_engine): keep legacy stream steps in sync and reject tool-call rewrites Each streaming step drops the response an earlier step's translation stored under request_data["response"], so a later legacy hook sees the stream as the steps before it left it instead of the first step's snapshot. A legacy replacement whose tool calls differ from the scanned chunks is now undeliverable like a text mismatch, so the original stream is released with a warning instead of delivering the text while dropping the tool-call change --- .../proxy/policy_engine/pipeline_executor.py | 17 ++-- .../policy_engine/test_pipeline_executor.py | 78 ++++++++++++++++--- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 089f0683823..b4f6da18d1c 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -150,8 +150,8 @@ class _LegacyHookStreamAdapter(CustomGuardrail): non-streaming hooks, an exception it raises ends the stream through the executor's fail/error classification, and a replacement response is re-scanned by the same translation so its texts reach the client through the translation's ended-stream write-back. A - replacement whose scanned texts do not line up with the originals is undeliverable, so the - executor releases the original chunks.""" + replacement whose scanned texts do not line up with the originals, or whose tool calls + differ from them, is undeliverable, so the executor releases the original chunks.""" def __init__( self, @@ -184,8 +184,12 @@ class _LegacyHookStreamAdapter(CustomGuardrail): return inputs scanned: Final = _text_snapshot(inputs.get("texts")) rescanned: Final = await self._rescan(replacement, logging_obj) - rewritten: Final = None if rescanned is None else rescanned.get("texts") - if scanned is None or rewritten is None or len(rewritten) != len(scanned): + if scanned is None or rescanned is None: + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + rewritten: Final = rescanned.get("texts") + if rewritten is None or len(rewritten) != len(scanned): + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + if _tool_call_shapes(rescanned.get("tool_calls")) != _tool_call_shapes(inputs.get("tool_calls")): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} return rewritten_inputs @@ -380,7 +384,9 @@ class PipelineExecutor: yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the originals and the step passes, so the client gets - the stream the merge base sent.""" + the stream the merge base sent. The response an earlier step's translation stored under + ``request_data["response"]`` is dropped first, so this step's hook sees the stream as + the steps before it left it.""" scanner: Final = ( callback if PipelineExecutor.supports_unified_execution(callback) @@ -389,6 +395,7 @@ class PipelineExecutor: observer: Final = _StreamRewriteObserver(scanner) deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites originals: Final = copy.deepcopy(streaming_chunks) + hook_input.pop("response", None) # rebind-ok: an earlier step's stored response goes so this step's is stored try: if deliver_rewrites: await endpoint_translation.process_output_streaming_response( diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index a44afc6ef96..860f83a06bb 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -675,7 +675,6 @@ async def test_guardrail_not_found_uses_on_fail(monkeypatch): ], ) - result = await PipelineExecutor.execute_steps( steps=pipeline.steps, mode=pipeline.mode, @@ -1298,8 +1297,8 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri class _LegacyHookGuardrail(CustomGuardrail): """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" - def __init__(self, replacement=None, raises=None): - super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + def __init__(self, replacement=None, raises=None, guardrail_name="masker"): + super().__init__(guardrail_name=guardrail_name, event_hook="post_call", default_on=True) self.replacement = replacement self.raises = raises self.calls = [] @@ -1339,7 +1338,7 @@ class _LegacyScanningTranslation: ): request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) outputs = await guardrail_to_apply.apply_guardrail( - inputs={"texts": [responses_so_far[0]["text"]]}, + inputs={"texts": [responses_so_far[0]["text"]], "tool_calls": [dict(responses_so_far[0]["tool_call"])]}, request_data=request_data, input_type="response", logging_obj=litellm_logging_obj, @@ -1350,8 +1349,11 @@ class _LegacyScanningTranslation: async def process_output_response( self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None ): + inputs = {"texts": list(response["texts"])} + if response.get("tool_calls"): + inputs["tool_calls"] = list(response["tool_calls"]) await guardrail_to_apply.apply_guardrail( - inputs={"texts": list(response["texts"])}, + inputs=inputs, request_data={"response": response}, input_type="response", logging_obj=litellm_logging_obj, @@ -1359,10 +1361,26 @@ class _LegacyScanningTranslation: return response +def _legacy_replacement(*texts, tool_calls=None): + return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} + + async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): - monkeypatch.setattr(litellm, "callbacks", [guardrail]) + return await _run_legacy_streaming_steps(monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error) + + +async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail="block", on_error="next"): + monkeypatch.setattr(litellm, "callbacks", list(guardrails)) return await PipelineExecutor.execute_steps( - steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail=on_fail, on_error=on_error)], + steps=[ + PipelineStep( + guardrail=guardrail.guardrail_name, + on_pass="next" if position + 1 < len(guardrails) else "allow", + on_fail=on_fail, + on_error=on_error, + ) + for position, guardrail in enumerate(guardrails) + ], mode="post_call", data={"model": "m"}, user_api_key_dict=MagicMock(), @@ -1376,7 +1394,7 @@ async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="bl @pytest.mark.asyncio @pytest.mark.parametrize("guardrail_class", [_LegacyHookGuardrail, _NativeHooksGuardrail]) async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypatch, caplog, guardrail_class): - guardrail = guardrail_class(replacement={"texts": ["[REWRITTEN] hello world"]}) + guardrail = guardrail_class(replacement=_legacy_replacement("[REWRITTEN] hello world")) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -1434,7 +1452,7 @@ async def test_streaming_step_takes_on_error_when_legacy_hook_crashes(monkeypatc @pytest.mark.asyncio async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up(monkeypatch, caplog): - guardrail = _LegacyHookGuardrail(replacement={"texts": ["split", "in two"]}) + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("split", "in two")) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -1442,3 +1460,45 @@ async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_that_changes_a_tool_call(monkeypatch, caplog): + masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} + guardrail = _LegacyHookGuardrail( + replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[masked_tool_call]) + ) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[])) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): + masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) + auditor = _LegacyHookGuardrail(replacement=None, guardrail_name="auditor") + chunks = [_chunk()] + + result = await _run_legacy_streaming_steps(monkeypatch, [masker, auditor], chunks, on_fail="next") + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass", "pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert [call["response"] for call in masker.calls] == [{"native": True, "text": "hello world"}] + assert [call["response"] for call in auditor.calls] == [{"native": True, "text": "[REWRITTEN] hello world"}] From 3ea65e761ee2f021e967c87202267cd4d70ff3f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:30:06 -0700 Subject: [PATCH 25/81] test(guardrails): cover Responses and Messages pipeline tool-call delivery --- .../proxy_logging/test_guardrail_pipeline.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index f753d2ab690..bf71780ad7b 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -25,6 +25,7 @@ from litellm.integrations.custom_guardrail import ( ModifyResponseException, ) from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines @@ -2185,3 +2186,106 @@ async def test_per_chunk_streaming_hook_runs_guardrail_whose_pipeline_cannot_str assert result is not None assert seen["count"] == 1 assert seen["response"] == "hello " + + +def _mask_tool_call_arguments(inputs: Dict[str, Any]) -> Dict[str, Any]: + return { + "tool_calls": [ + { + "id": stream_item_field(tool_call, "id"), + "type": "function", + "function": { + "name": stream_item_field(stream_item_field(tool_call, "function"), "name"), + "arguments": '{"fruit": "[MASKED]"}', + }, + } + for tool_call in inputs.get("tool_calls", []) + ] + } + + +def _anthropic_tool_use_sse_chunks() -> List[bytes]: + events = [ + ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "m", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup_fruit", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"fruit": "persim'}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": 'mon"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 2}}), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_delivers_tool_use_rewrite_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_mask_tool_call_arguments)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_tool_use_sse_chunks()), + request_data=data, + ) + ] + + raw = b"".join(delivered).decode() + assert '{\\"fruit\\": \\"[MASKED]\\"}' in raw + assert "persim" not in raw + assert '"name": "lookup_fruit"' in raw and '"id": "toolu_1"' in raw + assert '"stop_reason": "tool_use"' in raw + assert raw.count("event: content_block_delta") == 2 + + +def _responses_function_call_events() -> List[Dict[str, Any]]: + def item(arguments: str, status: str) -> Dict[str, Any]: + return { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 0, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 0, "delta": ' "persimmon"}'}, + {"type": "response.function_call_arguments.done", "item_id": "fc_1", "output_index": 0, "arguments": '{"fruit": "persimmon"}'}, + {"type": "response.output_item.done", "output_index": 0, "item": item('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": {"id": "resp_1", "created_at": 1, "model": "m", "output": [item('{"fruit": "persimmon"}', "completed")], "status": "completed"}, + }, + ] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_delivers_function_call_rewrite_on_responses_events( + proxy_logging, make_user_api_key_auth, monkeypatch +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_mask_tool_call_arguments)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/responses"), + response=_async_chunk_iter(_responses_function_call_events()), + request_data=data, + ) + ] + + assert [event["type"] for event in delivered] == [event["type"] for event in _responses_function_call_events()] + assert [event["delta"] for event in delivered if event["type"] == "response.function_call_arguments.delta"] == ['{"fruit": "[MASKED]"}', ""] + assert delivered[3]["arguments"] == '{"fruit": "[MASKED]"}' + assert delivered[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert delivered[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' + assert "persimmon" not in json.dumps(delivered) From ea427e33d8208034f934e52a7293f7f0bfa020ec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:44:14 -0700 Subject: [PATCH 26/81] fix(policy_engine): deliver legacy stream rewrites made in place A legacy post-call hook that changes the response it was handed and returns None (the model armor guardrail masks choice content that way) used to have that rewrite dropped on the streaming pipeline path, since the adapter only re-scanned a returned replacement. The adapter now re-scans the response it handed the hook when the hook returns None, so an in-place rewrite reaches the client through the same ended-stream write-back --- .../proxy/policy_engine/pipeline_executor.py | 13 ++++--- .../policy_engine/test_pipeline_executor.py | 37 +++++++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index b4f6da18d1c..45a113fcda2 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -148,8 +148,9 @@ class _LegacyHookStreamAdapter(CustomGuardrail): endpoint translation hands it the texts it scanned plus the assembled response under ``request_data["response"]``; the hook gets that response in the shape its route gives non-streaming hooks, an exception it raises ends the stream through the executor's - fail/error classification, and a replacement response is re-scanned by the same translation - so its texts reach the client through the translation's ended-stream write-back. A + fail/error classification, and the response it hands back, or the one it changed in place + and returned ``None`` for, is re-scanned by the same translation so its texts reach the + client through the translation's ended-stream write-back. A replacement whose scanned texts do not line up with the originals, or whose tool calls differ from them, is undeliverable, so the executor releases the original chunks.""" @@ -175,15 +176,17 @@ class _LegacyHookStreamAdapter(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: + hooked: Final = self.endpoint_translation.post_call_hook_response(request_data.get("response")) replacement: Final = await self.inner.async_post_call_success_hook( data=request_data, user_api_key_dict=self.user_api_key_dict, - response=self.endpoint_translation.post_call_hook_response(request_data.get("response")), + response=hooked, ) - if replacement is None: + rewrite: Final = hooked if replacement is None else replacement + if rewrite is None: return inputs scanned: Final = _text_snapshot(inputs.get("texts")) - rescanned: Final = await self._rescan(replacement, logging_obj) + rescanned: Final = await self._rescan(rewrite, logging_obj) if scanned is None or rescanned is None: raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten: Final = rescanned.get("texts") diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 860f83a06bb..ae7754e5821 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1297,16 +1297,19 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri class _LegacyHookGuardrail(CustomGuardrail): """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" - def __init__(self, replacement=None, raises=None, guardrail_name="masker"): + def __init__(self, replacement=None, raises=None, guardrail_name="masker", rewrite_in_place=None): super().__init__(guardrail_name=guardrail_name, event_hook="post_call", default_on=True) self.replacement = replacement self.raises = raises + self.rewrite_in_place = rewrite_in_place self.calls = [] async def async_post_call_success_hook(self, data, user_api_key_dict, response): self.calls.append({"data": data, "user_api_key_dict": user_api_key_dict, "response": response}) if self.raises is not None: raise self.raises + if self.rewrite_in_place is not None: + response["text"] = self.rewrite_in_place return self.replacement @@ -1325,7 +1328,7 @@ class _LegacyScanningTranslation: delivers_ended_stream_text_rewrites = True def post_call_hook_response(self, response): - return {"native": True, "text": response["text"]} + return {"native": True, "text": response["text"], "tool_calls": response["tool_calls"]} async def process_output_streaming_response( self, @@ -1336,7 +1339,9 @@ class _LegacyScanningTranslation: request_data=None, deliver_ended_stream_rewrites=False, ): - request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) + request_data.setdefault( + "response", {"text": responses_so_far[0]["text"], "tool_calls": [dict(responses_so_far[0]["tool_call"])]} + ) outputs = await guardrail_to_apply.apply_guardrail( inputs={"texts": [responses_so_far[0]["text"]], "tool_calls": [dict(responses_so_far[0]["tool_call"])]}, request_data=request_data, @@ -1349,7 +1354,7 @@ class _LegacyScanningTranslation: async def process_output_response( self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None ): - inputs = {"texts": list(response["texts"])} + inputs = {"texts": [response["text"]] if "text" in response else list(response["texts"])} if response.get("tool_calls"): inputs["tool_calls"] = list(response["tool_calls"]) await guardrail_to_apply.apply_guardrail( @@ -1361,6 +1366,10 @@ class _LegacyScanningTranslation: return response +def _native(text): + return {"native": True, "text": text, "tool_calls": [_chunk()["tool_call"]]} + + def _legacy_replacement(*texts, tool_calls=None): return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} @@ -1403,12 +1412,26 @@ async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypa assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass"] assert chunks[0]["text"] == "[REWRITTEN] hello world" - assert [call["response"] for call in guardrail.calls] == [{"native": True, "text": "hello world"}] + assert [call["response"] for call in guardrail.calls] == [_native("hello world")] assert guardrail.calls[0]["data"]["model"] == "m" assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] assert not any("discarded" in record.getMessage() for record in caplog.records) +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_rewrite_made_in_place(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(rewrite_in_place="[REWRITTEN] hello world") + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert not any("discarded" in record.getMessage() for record in caplog.records) + + @pytest.mark.asyncio async def test_streaming_step_passes_untouched_when_legacy_hook_returns_none(monkeypatch, caplog): guardrail = _LegacyHookGuardrail(replacement=None) @@ -1500,5 +1523,5 @@ async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(mon assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass", "pass"] assert chunks[0]["text"] == "[REWRITTEN] hello world" - assert [call["response"] for call in masker.calls] == [{"native": True, "text": "hello world"}] - assert [call["response"] for call in auditor.calls] == [{"native": True, "text": "[REWRITTEN] hello world"}] + assert [call["response"] for call in masker.calls] == [_native("hello world")] + assert [call["response"] for call in auditor.calls] == [_native("[REWRITTEN] hello world")] From ce7cec1a3635bdb66292daa45ec9fe39f37bbffa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:48:22 -0700 Subject: [PATCH 27/81] fix(policy_engine): keep discarded stream rewrites out of the applied-guardrails header A streaming step whose rewrite the executor threw away (a tool-call rewrite, a text rewrite the translation cannot write back, or one the adapter refused) still marked its guardrail as applied, so the header claimed an output the client never received. The step now returns right after releasing the original chunks, which leaves the header as the merge base sent it --- litellm/proxy/policy_engine/pipeline_executor.py | 10 ++++++---- .../proxy/policy_engine/test_pipeline_executor.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 45a113fcda2..9b0d1e60839 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -387,7 +387,8 @@ class PipelineExecutor: yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the originals and the step passes, so the client gets - the stream the merge base sent. The response an earlier step's translation stored under + the stream the merge base sent, and the guardrail stays out of the applied-guardrails + header since its output never reached the client. The response an earlier step's translation stored under ``request_data["response"]`` is dropped first, so this step's hook sees the stream as the steps before it left it.""" scanner: Final = ( @@ -419,9 +420,10 @@ class PipelineExecutor: ) except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) - else: - if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): - _release_original_chunks(step.guardrail, streaming_chunks, originals) + return + if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): + _release_original_chunks(step.guardrail, streaming_chunks, originals) + return if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index ae7754e5821..2af318c88cd 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1146,6 +1146,7 @@ def _assert_passed_with_discard_warning(result, caplog): assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass"] assert any("'masker'" in record.getMessage() and "discarded" in record.getMessage() for record in caplog.records) + assert "masker" not in ((result.modified_data or {}).get("metadata") or {}).get("applied_guardrails", []) @pytest.mark.asyncio From 9cb5d9b76c0c6b6e7a20e852e9673c9ea410c3f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:55:19 -0700 Subject: [PATCH 28/81] fix(proxy): gate streaming pipelines on a route-resolved guardrail translation The per-chunk hook skipped pipeline-managed guardrails whenever the request route was empty, while the gated stream could still fail to resolve a translation and release the buffered stream ungoverned. The gate now needs a translation resolved from the route, the iterator hook resolves it once and hands it to the gated stream, and the ungoverned release branch is gone. --- litellm/proxy/utils.py | 27 ++++------ .../proxy_logging/test_guardrail_pipeline.py | 53 +++++++++++++++---- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c38eb900a05..37d077d607c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -192,6 +192,7 @@ if TYPE_CHECKING: from prisma.types import HttpConfig from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.models.team import LiteLLM_TeamTableCachedObj from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -559,7 +560,7 @@ def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> def _route_supports_streaming_pipelines(user_api_key_dict: UserAPIKeyAuth) -> bool: - return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None + return resolve_endpoint_translation(user_api_key_dict, None) is not None def _stream_gated_guardrail_names( @@ -3435,12 +3436,16 @@ class ProxyLogging: ), ) - if post_call_pipelines: + pipeline_translation: Final = ( + resolve_endpoint_translation(user_api_key_dict, None) if post_call_pipelines else None + ) + if pipeline_translation is not None: current_response = self._pipeline_gated_stream( response=current_response, user_api_key_dict=user_api_key_dict, request_data=request_data, pipelines=post_call_pipelines, + translation=pipeline_translation, ) try: @@ -3464,6 +3469,7 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, request_data: dict, # mutable-ok: same request-payload shape the hooks mutate pipelines: "tuple[tuple[str, GuardrailPipeline], ...]", + translation: "tuple[str, BaseTranslation]", ) -> "AsyncGenerator[Any, None]": """ Execute post_call policy pipelines against a streamed response. @@ -3478,9 +3484,8 @@ class ProxyLogging: rewritten chunks, so rewrites chain). A rewrite the translation cannot deliver yet (one on a route without write-back, or a shape the route refuses) is discarded by the executor and the original chunks are - released, as is a buffered shape no translation resolves; a block or - modify_response terminates with the translation's block chunks or the - raised error. + released; a block or modify_response terminates with the translation's + block chunks or the raised error. """ buffered: Final[list[object]] = [] # mutable-ok: accumulates the stream before the pipeline verdict async for item in response: @@ -3488,17 +3493,7 @@ class ProxyLogging: if not buffered: return - resolved: Final = resolve_endpoint_translation(user_api_key_dict, buffered[0]) - if resolved is None: - verbose_proxy_logger.warning( - "Policies with post_call guardrail pipelines cannot scan this streaming response shape yet; " - "the stream is released ungoverned by them: %s", - ", ".join(policy_name for policy_name, _pipeline in pipelines), - ) - for buffered_item in buffered: - yield buffered_item - return - call_type, endpoint_translation = resolved + call_type, endpoint_translation = translation for policy_name, pipeline in pipelines: result: PipelineExecutionResult = await PipelineExecutor.execute_steps( diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index bf71780ad7b..3b59bbac3fd 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1969,28 +1969,61 @@ async def test_streaming_iterator_hook_pipeline_releases_stream_echoed_in_anothe @pytest.mark.asyncio -async def test_streaming_iterator_hook_pipeline_releases_originals_on_unresolvable_response_shape( +async def test_streaming_iterator_hook_skips_pipeline_and_warns_without_request_route( proxy_logging, make_user_api_key_auth, monkeypatch, caplog ): seen: Dict[str, Any] = {} monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)]) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) - chunks = [object(), object()] - delivered: List[Any] = [] + chunks = _stream_chunks() with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - async for item in proxy_logging.async_post_call_streaming_iterator_hook( - user_api_key_dict=make_user_api_key_auth(), - response=_async_chunk_iter(chunks), - request_data=data, - ): - delivered.append(item) + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(), + response=_async_chunk_iter(chunks), + request_data=data, + ) + ] assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True] assert len(delivered) == 2 assert seen.get("count") is None - assert any("response-governance" in message and "shape" in message for message in _warnings(caplog)) + assert any("response-governance" in message and "route None" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_per_chunk_streaming_hook_runs_pipeline_managed_guardrail_without_request_route( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + + class UnifiedRecordingGuardrail(CustomGuardrail): + async def async_post_call_streaming_hook(self, user_api_key_dict, response): + seen[self.guardrail_name] = seen.get(self.guardrail_name, 0) + 1 + return None + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + monkeypatch.setattr( + litellm, + "callbacks", + [UnifiedRecordingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)], + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + result = await proxy_logging.async_post_call_streaming_hook( + data=data, + response=_stream_chunks()[0], + user_api_key_dict=make_user_api_key_auth(), + ) + + assert result is not None + assert seen["gr-post"] == 1 def _anthropic_sse_chunks() -> List[bytes]: From 88d2d775534457b3397c9332ca968b7a8707a161 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:59:55 -0700 Subject: [PATCH 29/81] feat(hosted_vllm): add image edit support Register a HostedVLLMImageEditConfig so hosted_vllm/ deployments route POST /v1/images/edits to the vLLM-Omni OpenAI-compatible endpoint instead of failing with 'image edit is not supported for hosted_vllm' before any request is sent --- .../llms/hosted_vllm/image_edit/__init__.py | 9 ++ .../hosted_vllm/image_edit/transformation.py | 42 +++++++ litellm/utils.py | 4 + ...t_hosted_vllm_image_edit_transformation.py | 114 ++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 litellm/llms/hosted_vllm/image_edit/__init__.py create mode 100644 litellm/llms/hosted_vllm/image_edit/transformation.py create mode 100644 tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py diff --git a/litellm/llms/hosted_vllm/image_edit/__init__.py b/litellm/llms/hosted_vllm/image_edit/__init__.py new file mode 100644 index 00000000000..27e005e8a0d --- /dev/null +++ b/litellm/llms/hosted_vllm/image_edit/__init__.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .transformation import HostedVLLMImageEditConfig + +__all__ = ("HostedVLLMImageEditConfig",) + + +def get_hosted_vllm_image_edit_config(model: str) -> BaseImageEditConfig: + return HostedVLLMImageEditConfig() diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py new file mode 100644 index 00000000000..bd27c962da8 --- /dev/null +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -0,0 +1,42 @@ +"""Image edits for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/images/edits).""" + +from typing import Final + +from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig +from litellm.secret_managers.main import get_secret_str + + +class HostedVLLMImageEditConfig(OpenAIImageEditConfig): + """ + vLLM-Omni images edits API follows the OpenAI multipart contract. + + https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/images_api/ + """ + + def validate_environment( + self, + headers: dict, # mutable-ok: BaseImageEditConfig contract + model: str, + api_key: str | None = None, + litellm_params: dict | None = None, # mutable-ok: BaseImageEditConfig contract + api_base: str | None = None, + ) -> dict: # mutable-ok: BaseImageEditConfig contract + resolved_key: Final = api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key" + return {**headers, "Authorization": f"Bearer {resolved_key}"} # mutable-ok: httpx headers are a dict + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: BaseImageEditConfig contract + ) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError( + "api_base not set for Hosted VLLM images edits API. " + "Set via api_base parameter or HOSTED_VLLM_API_BASE environment variable" + ) + trimmed: Final = resolved_api_base.rstrip("/") + if trimmed.endswith("/v1"): + return f"{trimmed}/images/edits" + return f"{trimmed}/v1/images/edits" diff --git a/litellm/utils.py b/litellm/utils.py index 141ec323776..0f2b2f9042d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9239,6 +9239,10 @@ class ProviderConfigManager: from litellm.llms.openai.image_edit import get_openai_image_edit_config return get_openai_image_edit_config(model=model) + if LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py new file mode 100644 index 00000000000..e1167b92553 --- /dev/null +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -0,0 +1,114 @@ +"""Tests for hosted_vllm image edits (vLLM-Omni /v1/images/edits).""" + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config +from litellm.llms.hosted_vllm.image_edit.transformation import HostedVLLMImageEditConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +PNG_BYTES = b"\x89PNG\r\n\x1a\nfakepng" +MODEL = "Qwen/Qwen-Image-Edit-2511" + + +@pytest.fixture(autouse=True) +def _clear_hosted_vllm_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + + +def test_provider_config_registration(): + config = ProviderConfigManager.get_provider_image_edit_config( + model=f"hosted_vllm/{MODEL}", + provider=LlmProviders.HOSTED_VLLM, + ) + + assert isinstance(config, HostedVLLMImageEditConfig) + assert isinstance(get_hosted_vllm_image_edit_config(MODEL), HostedVLLMImageEditConfig) + + +@pytest.mark.parametrize( + "api_base", + ["http://localhost:8091", "http://localhost:8091/", "http://localhost:8091/v1", "http://localhost:8091/v1/"], +) +def test_get_complete_url_appends_images_edits(api_base: str): + config = HostedVLLMImageEditConfig() + + assert ( + config.get_complete_url(model=MODEL, api_base=api_base, litellm_params={}) + == "http://localhost:8091/v1/images/edits" + ) + + +def test_get_complete_url_falls_back_to_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm-omni:8000/v1") + config = HostedVLLMImageEditConfig() + + assert ( + config.get_complete_url(model=MODEL, api_base=None, litellm_params={}) + == "http://vllm-omni:8000/v1/images/edits" + ) + + +def test_get_complete_url_requires_api_base(): + config = HostedVLLMImageEditConfig() + + with pytest.raises(ValueError, match="api_base not set"): + config.get_complete_url(model=MODEL, api_base=None, litellm_params={}) + + +def test_validate_environment_defaults_to_fake_api_key(): + headers = HostedVLLMImageEditConfig().validate_environment(headers={}, model=MODEL) + + assert headers == {"Authorization": "Bearer fake-api-key"} + + +def test_validate_environment_uses_provided_api_key_and_keeps_headers(): + headers = HostedVLLMImageEditConfig().validate_environment( + headers={"X-Test": "1"}, + model=MODEL, + api_key="my-custom-key", + ) + + assert headers == {"X-Test": "1", "Authorization": "Bearer my-custom-key"} + + +def test_validate_environment_falls_back_to_env_api_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "env-key") + + headers = HostedVLLMImageEditConfig().validate_environment(headers={}, model=MODEL) + + assert headers["Authorization"] == "Bearer env-key" + + +def test_image_edit_posts_multipart_to_vllm_omni(): + captured: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + + response = litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + api_key="test-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))), + seed=42, + ) + + assert response.data + assert len(captured) == 1 + request = captured[0] + assert str(request.url) == "http://localhost:8091/v1/images/edits" + assert request.headers["authorization"] == "Bearer test-key" + assert request.headers["content-type"].startswith("multipart/form-data") + assert b'name="image[]"' in request.content + assert PNG_BYTES in request.content + assert f'name="model"\r\n\r\n{MODEL}'.encode() in request.content + assert b'name="prompt"\r\n\r\nadd a hat' in request.content + assert b'name="seed"\r\n\r\n42' in request.content From 260097b2dde7f12f77784892e96e9956b0b19cc1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:16:34 -0700 Subject: [PATCH 30/81] refactor(hosted_vllm): drop redundant image edit docstrings --- litellm/llms/hosted_vllm/image_edit/transformation.py | 8 -------- .../test_hosted_vllm_image_edit_transformation.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py index bd27c962da8..fa8c1dbfcfd 100644 --- a/litellm/llms/hosted_vllm/image_edit/transformation.py +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -1,5 +1,3 @@ -"""Image edits for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/images/edits).""" - from typing import Final from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig @@ -7,12 +5,6 @@ from litellm.secret_managers.main import get_secret_str class HostedVLLMImageEditConfig(OpenAIImageEditConfig): - """ - vLLM-Omni images edits API follows the OpenAI multipart contract. - - https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/images_api/ - """ - def validate_environment( self, headers: dict, # mutable-ok: BaseImageEditConfig contract diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py index e1167b92553..5646f3d9d13 100644 --- a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -1,5 +1,3 @@ -"""Tests for hosted_vllm image edits (vLLM-Omni /v1/images/edits).""" - import httpx import pytest From 9e67c083092da5e20b85ed87e935d0af168cc66d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:16:39 -0700 Subject: [PATCH 31/81] fix(policy_engine): keep tool-only legacy streams deliverable A Messages stream that ends with only tool_use blocks reaches the legacy step without a texts key, while the non-streaming rescan of the same response sends an empty list. Compare both as empty and keep the stream when the hook left the tool calls alone. --- .../proxy/policy_engine/pipeline_executor.py | 15 +++- .../policy_engine/test_pipeline_executor.py | 84 ++++++++++++++++++- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9b0d1e60839..52985d49c7e 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -70,6 +70,10 @@ def _text_snapshot(texts: Sequence[str] | None) -> tuple[str, ...] | None: return None if texts is None else tuple(texts) +def _scanned_texts(texts: Sequence[str] | None) -> tuple[str, ...]: + return tuple(texts or ()) + + def _tool_call_shapes(tool_calls: Sequence[object] | None) -> tuple[tuple[object, object], ...] | None: return None if tool_calls is None else tuple(_tool_call_shape(tool_call) for tool_call in tool_calls) @@ -152,7 +156,9 @@ class _LegacyHookStreamAdapter(CustomGuardrail): and returned ``None`` for, is re-scanned by the same translation so its texts reach the client through the translation's ended-stream write-back. A replacement whose scanned texts do not line up with the originals, or whose tool calls - differ from them, is undeliverable, so the executor releases the original chunks.""" + differ from them, is undeliverable, so the executor releases the original chunks. A stream + that carried no text to scan, such as a tool-only Anthropic message, stays deliverable as + long as the hook left the tool calls alone.""" def __init__( self, @@ -185,15 +191,16 @@ class _LegacyHookStreamAdapter(CustomGuardrail): rewrite: Final = hooked if replacement is None else replacement if rewrite is None: return inputs - scanned: Final = _text_snapshot(inputs.get("texts")) rescanned: Final = await self._rescan(rewrite, logging_obj) - if scanned is None or rescanned is None: + if rescanned is None: raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten: Final = rescanned.get("texts") - if rewritten is None or len(rewritten) != len(scanned): + if len(_scanned_texts(rewritten)) != len(_scanned_texts(inputs.get("texts"))): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") if _tool_call_shapes(rescanned.get("tool_calls")) != _tool_call_shapes(inputs.get("tool_calls")): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + if not rewritten: + return inputs rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} return rewritten_inputs diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 2af318c88cd..98674843c73 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1367,6 +1367,44 @@ class _LegacyScanningTranslation: return response +class _ToolOnlyLegacyScanningTranslation(_LegacyScanningTranslation): + """Like the Messages handler on a tool-only message: the ended-stream scan omits "texts" from + the inputs, while the non-streaming scan of the same response sends an empty list.""" + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + request_data.setdefault("response", {"text": "", "tool_calls": [dict(responses_so_far[0]["tool_call"])]}) + await guardrail_to_apply.apply_guardrail( + inputs={"tool_calls": [dict(responses_so_far[0]["tool_call"])]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return responses_so_far + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": [], "tool_calls": list(response.get("tool_calls") or [])}, + request_data={"response": response}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return response + + +def _tool_only_chunk(): + return {"text": "", "tool_call": _chunk()["tool_call"]} + + def _native(text): return {"native": True, "text": text, "tool_calls": [_chunk()["tool_call"]]} @@ -1375,11 +1413,17 @@ def _legacy_replacement(*texts, tool_calls=None): return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} -async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): - return await _run_legacy_streaming_steps(monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error) +async def _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, on_fail="block", on_error="next", translation=None +): + return await _run_legacy_streaming_steps( + monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error, translation=translation + ) -async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail="block", on_error="next"): +async def _run_legacy_streaming_steps( + monkeypatch, guardrails, chunks, on_fail="block", on_error="next", translation=None +): monkeypatch.setattr(litellm, "callbacks", list(guardrails)) return await PipelineExecutor.execute_steps( steps=[ @@ -1397,7 +1441,7 @@ async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail=" call_type="completion", policy_name="p", streaming_chunks=chunks, - endpoint_translation=_LegacyScanningTranslation(), + endpoint_translation=_LegacyScanningTranslation() if translation is None else translation, ) @@ -1513,6 +1557,38 @@ async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls( assert chunks == [_chunk()] +@pytest.mark.asyncio +async def test_streaming_step_passes_a_tool_only_stream_the_legacy_hook_left_alone(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=None) + chunks = [_tool_only_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert chunks == [_tool_only_chunk()] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only_stream(monkeypatch, caplog): + masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement(tool_calls=[masked_tool_call])) + chunks = [_tool_only_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() + ) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_tool_only_chunk()] + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From 359c26aa1d1403b0dc10c2d7a47b984d13c468c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:27:46 -0700 Subject: [PATCH 32/81] test(policy_engine): cover the legacy stream paths with no response and no rescan A translation that hands the hook no assembled response leaves the stream as it is, and a rewrite the translation cannot rescan is released as the original stream with a warning. --- .../policy_engine/test_pipeline_executor.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 98674843c73..6de7f7b593d 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1589,6 +1589,68 @@ async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only assert chunks == [_tool_only_chunk()] +class _ResponselessLegacyScanningTranslation(_LegacyScanningTranslation): + """Like a handler that never stores the assembled response under request_data["response"].""" + + def post_call_hook_response(self, response): + return response + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": [responses_so_far[0]["text"]]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return responses_so_far + + +class _UnscannableRewriteTranslation(_LegacyScanningTranslation): + """Like the chat handler on a response whose choices are plain dicts: the non-streaming scan + never hands anything to the guardrail.""" + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + return response + + +@pytest.mark.asyncio +async def test_streaming_step_leaves_the_stream_alone_when_the_hook_gets_no_response(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail() + chunks = [_chunk()] + + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ResponselessLegacyScanningTranslation() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert guardrail.calls[0]["response"] is None + assert chunks == [_chunk()] + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_a_legacy_rewrite_the_translation_cannot_rescan(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("hello [MASKED]")) + chunks = [_chunk()] + + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks, translation=_UnscannableRewriteTranslation()) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From 113350756595b7954d5f830f8a5454895de12526 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:46:55 -0700 Subject: [PATCH 33/81] refactor: type the buffered stream rewrite helpers without Any --- .../llms/anthropic/chat/guardrail_translation/handler.py | 8 ++++---- .../responses/test_openai_responses_guardrail_handler.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index cbbccac17f3..6f966ae1a02 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,7 +13,7 @@ Pattern Overview: """ import json -from collections.abc import Mapping, Sequence +from collections.abc import Mapping, MutableSequence, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat @@ -1262,7 +1262,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_ended_stream_text_rewrite( - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place rewritten_text: str, ) -> None: """Deliver an ended-stream guardrail text rewrite by rewriting the @@ -1284,7 +1284,7 @@ class AnthropicMessagesHandler(BaseTranslation): @classmethod def _write_ended_stream_tool_call_rewrites( cls, - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place *, pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], post_guardrail_tool_calls: tuple[_ToolCallShape, ...], @@ -1346,7 +1346,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_ended_stream_events( - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place rewrite_event: _SSEEventRewriter, ) -> None: """Replace every buffered event ``rewrite_event`` returns a rewrite for, in diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index fa1e969b277..6ed4ec6618f 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -17,6 +17,7 @@ from fastapi import HTTPException from openai.types.responses import ResponseFunctionToolCall from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, @@ -1238,7 +1239,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[Any] = None, + logging_obj: LiteLLMLoggingObj | None = None, ) -> GenericGuardrailAPIInputs: tool_calls = [ {**tool_call, "function": {**tool_call["function"], "arguments": '{"fruit": "[MASKED]"}'}} From ddedb4867b8cedea4ac2223dbc636f813e8bf997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:09:44 -0700 Subject: [PATCH 34/81] fix: discard a streamed rewrite that drops or adds a tool call A guardrail that removes or adds a tool call on an ended stream used to be silently ignored: every handler substitutes the original list on a count mismatch and the executor skipped its observer once the translation could deliver rewrites. The executor now tracks the count change on the observer and releases the original chunks with the discard warning on every translation, matching what the merge base did for any tool call rewrite --- .../proxy/policy_engine/pipeline_executor.py | 20 ++++-- .../policy_engine/test_pipeline_executor.py | 23 +++++- .../proxy_logging/test_guardrail_pipeline.py | 71 +++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index c870bd7d2ec..a51468cc0fb 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -78,6 +78,10 @@ def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | Non return sent is not None and returned is not None and returned != sent +def _changed_count(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> bool: + return sent is not None and returned is not None and len(returned) != len(sent) + + _GuardrailMethodT = TypeVar("_GuardrailMethodT", bound=Callable[..., object]) @@ -91,8 +95,9 @@ class _StreamRewriteObserver(CustomGuardrail): guardrail. It records whether the guardrail returned different output than it was given, which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text and tool-call rewrites are deliverable on translations that write them back across the - buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation - are discarded by the executor, which releases the original chunks. + buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation, + and a rewrite that drops or adds a tool call on any translation, are discarded by the + executor, which releases the original chunks. The inner guardrail's ``apply_guardrail`` already records the guardrail information and span, so the observer's stays out of ``log_guardrail_information``.""" @@ -101,6 +106,7 @@ class _StreamRewriteObserver(CustomGuardrail): self.inner: Final = inner self.rewrote_texts = False self.rewrote_tool_calls = False + self.changed_tool_call_count = False def structured_messages_cover_full_request(self) -> bool: return self.inner.structured_messages_cover_full_request() @@ -118,9 +124,11 @@ class _StreamRewriteObserver(CustomGuardrail): outputs: Final = await self.inner.apply_guardrail( inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj ) + returned_tool_shapes: Final = _tool_call_shapes(outputs.get("tool_calls")) self.rewrote_texts = self.rewrote_texts or _rewrote(sent_texts, _text_snapshot(outputs.get("texts"))) - self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote( - sent_tool_shapes, _tool_call_shapes(outputs.get("tool_calls")) + self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote(sent_tool_shapes, returned_tool_shapes) + self.changed_tool_call_count = self.changed_tool_call_count or _changed_count( + sent_tool_shapes, returned_tool_shapes ) return outputs @@ -325,7 +333,9 @@ class PipelineExecutor: except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) else: - if not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls): + if observer.changed_tool_call_count or ( + not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls) + ): _release_original_chunks(step.guardrail, streaming_chunks, originals) if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 5d042d3c1ad..16401ccdaad 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1106,7 +1106,8 @@ class _WritingTranslation: logging_obj=litellm_logging_obj, ) responses_so_far[0]["text"] = outputs["texts"][0] - responses_so_far[0]["tool_call"] = outputs["tool_calls"][0] + if len(outputs["tool_calls"]) == 1: + responses_so_far[0]["tool_call"] = outputs["tool_calls"][0] return responses_so_far @@ -1242,6 +1243,26 @@ async def test_streaming_step_delivers_tool_call_rewrite_through_writing_transla assert not any("discarded" in record.getMessage() for record in caplog.records) +class _ToolCallDroppingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return {**inputs, "texts": ["hello [MASKED]"], "tool_calls": []} + + +@pytest.mark.asyncio +async def test_streaming_step_discards_whole_rewrite_when_guardrail_drops_a_tool_call(monkeypatch, caplog): + monkeypatch.setattr(litellm, "callbacks", [_ToolCallDroppingGuardrail()]) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_streaming_step(_WritingTranslation(), chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + @pytest.mark.asyncio async def test_streaming_step_discards_tool_call_rewrite_when_translation_lacks_write_back(monkeypatch, caplog): monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 3b59bbac3fd..d46881d045b 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -2322,3 +2322,74 @@ async def test_streaming_iterator_hook_pipeline_delivers_function_call_rewrite_o assert delivered[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' assert delivered[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' assert "persimmon" not in json.dumps(delivered) + + +def _drop_tool_calls(inputs: Dict[str, Any]) -> Dict[str, Any]: + return {"tool_calls": []} + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_chat_chunks( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_tool_call_stream_chunks()), + request_data=data, + ) + ] + + assert delivered[0].choices[0].delta.tool_calls[0].function.arguments == '{"ssn": "123"}' + assert delivered[1].choices[0].finish_reason == "tool_calls" + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_tool_use_sse_chunks()), + request_data=data, + ) + ] + + assert delivered == _anthropic_tool_use_sse_chunks() + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_responses_events( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/responses"), + response=_async_chunk_iter(_responses_function_call_events()), + request_data=data, + ) + ] + + assert delivered == _responses_function_call_events() + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) From c66ae07e3bb97e875d88096458e5de80b351828c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:11:01 -0700 Subject: [PATCH 35/81] fix(hosted_vllm): reject image edit params vLLM-Omni ignores --- .../hosted_vllm/image_edit/transformation.py | 9 ++++ litellm/utils.py | 2 +- ...t_hosted_vllm_image_edit_transformation.py | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py index fa8c1dbfcfd..3b8cc437168 100644 --- a/litellm/llms/hosted_vllm/image_edit/transformation.py +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -3,8 +3,17 @@ from typing import Final from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str +PARAMS_VLLM_OMNI_DOES_NOT_ACCEPT: Final = frozenset({"mask", "quality", "input_fidelity"}) + class HostedVLLMImageEditConfig(OpenAIImageEditConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseImageEditConfig contract + return [ # mutable-ok: BaseImageEditConfig returns list + param + for param in super().get_supported_openai_params(model) + if param not in PARAMS_VLLM_OMNI_DOES_NOT_ACCEPT + ] + def validate_environment( self, headers: dict, # mutable-ok: BaseImageEditConfig contract diff --git a/litellm/utils.py b/litellm/utils.py index 0f2b2f9042d..fc530f14e28 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9239,7 +9239,7 @@ class ProviderConfigManager: from litellm.llms.openai.image_edit import get_openai_image_edit_config return get_openai_image_edit_config(model=model) - if LlmProviders.HOSTED_VLLM == provider: + elif LlmProviders.HOSTED_VLLM == provider: from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config return get_hosted_vllm_image_edit_config(model=model) diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py index 5646f3d9d13..bc0ea23e249 100644 --- a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -110,3 +110,46 @@ def test_image_edit_posts_multipart_to_vllm_omni(): assert f'name="model"\r\n\r\n{MODEL}'.encode() in request.content assert b'name="prompt"\r\n\r\nadd a hat' in request.content assert b'name="seed"\r\n\r\n42' in request.content + + +@pytest.mark.parametrize("param", ["mask", "quality", "input_fidelity"]) +def test_params_vllm_omni_ignores_are_not_advertised(param: str): + supported = HostedVLLMImageEditConfig().get_supported_openai_params(MODEL) + + assert param not in supported + assert {"image", "prompt", "n", "size", "response_format", "background", "user"} <= set(supported) + + +def test_image_edit_rejects_quality_unless_dropped(): + captured: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))) + + with pytest.raises(litellm.UnsupportedParamsError, match="quality"): + litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + client=client, + quality="low", + ) + assert captured == [] + + litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + client=client, + quality="low", + drop_params=True, + ) + + assert len(captured) == 1 + assert b'name="quality"' not in captured[0].content + assert b'name="prompt"\r\n\r\nadd a hat' in captured[0].content From 8d040d89e68c075b105a6369e5d95fdc7c1ba8f0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:39:56 -0700 Subject: [PATCH 36/81] fix(policy_engine): keep legacy hooks off streams their route cannot assemble and off guardrails with their own iterator hook The streaming pipeline step only takes a post-call hook on routes whose translation assembles the streamed response (chat completions, Responses, Messages). On /v1/completions, the Gemini streamGenerateContent route, and A2A streams the pipeline is skipped with the merge-base warning and the hook runs on its own afterwards, instead of getting a None response while the header says the guardrail ran. A guardrail that overrides async_post_call_streaming_iterator_hook next to its post-call hook keeps its native per-chunk path rather than running buffered through the adapter --- .../chat/guardrail_translation/handler.py | 1 + .../guardrail_translation/base_translation.py | 7 ++ .../chat/guardrail_translation/handler.py | 1 + .../guardrail_translation/handler.py | 1 + .../proxy/policy_engine/pipeline_executor.py | 14 ++-- litellm/proxy/utils.py | 60 +++++++++++---- .../policy_engine/test_pipeline_executor.py | 46 +++-------- .../proxy_logging/test_guardrail_pipeline.py | 77 ++++++++++++++++++- 8 files changed, 148 insertions(+), 59 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index f61cfe58e80..b83235f660f 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -171,6 +171,7 @@ class AnthropicMessagesHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def __init__(self): super().__init__() diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index bef472b2882..375b22ff5f6 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -60,6 +60,13 @@ class BaseTranslation(ABC): text rewrites on every other translation, are undeliverable: the pipeline executor discards them and releases the original chunks.""" + assembles_streamed_response: ClassVar[bool] = False + """Whether ``process_output_streaming_response`` stores the assembled response of an + ended stream under ``request_data["response"]`` before scanning it, the way the chat, + Responses, and Messages translations do. A streaming pipeline runs a guardrail that only + has the legacy post-call hook against that response, so on a translation without it such + a guardrail keeps running on its own.""" + def post_call_hook_response(self, response: object) -> object: """The ``response`` this endpoint's non-streaming post-call hooks receive, derived from the object the translation stores under ``request_data["response"]`` while scanning an diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 80292aef2cf..b9b88061fec 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -79,6 +79,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index b0f79552bc5..e0ba80706df 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -341,6 +341,7 @@ class OpenAIResponsesHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 52985d49c7e..ab5ad01fda9 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -556,12 +556,14 @@ class PipelineExecutor: @staticmethod def supports_streaming_execution(callback: CustomGuardrail) -> bool: """Whether a streaming pipeline step can run this guardrail against the buffered - stream: through the unified path, or through its own post-call hook on the - assembled response. A guardrail with neither (one that only rewrites the stream - through its iterator hook) has to keep running on its own.""" - return ( - PipelineExecutor.supports_unified_execution(callback) - or type(callback).async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + stream: through the unified path, or through its post-call hook on the assembled + response when that hook is its only streaming path. A guardrail with its own + streaming iterator hook, or with neither hook, keeps running on its own.""" + callback_type: Final = type(callback) + return PipelineExecutor.supports_unified_execution(callback) or ( + callback_type.async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + and callback_type.async_post_call_streaming_iterator_hook + is CustomLogger.async_post_call_streaming_iterator_hook ) @staticmethod diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 23260785b97..67a579c4d15 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -192,6 +192,7 @@ if TYPE_CHECKING: from prisma.types import HttpConfig from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.models.team import LiteLLM_TeamTableCachedObj from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -517,9 +518,17 @@ def _merge_pipeline_metadata_writes( _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key)) -def _pipeline_step_supports_streaming(guardrail_name: str) -> bool: +def _pipeline_step_supports_streaming(guardrail_name: str, translation: "BaseTranslation | None") -> bool: callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name) - return callback is not None and PipelineExecutor.supports_streaming_execution(callback) + if callback is None: + return False + if PipelineExecutor.supports_unified_execution(callback): + return True + return ( + translation is not None + and type(translation).assembles_streamed_response + and PipelineExecutor.supports_streaming_execution(callback) + ) def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]: @@ -541,42 +550,57 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No ) -def _pipeline_unsupported_streaming_guardrails(pipeline: "GuardrailPipeline") -> tuple[str, ...]: +def _pipeline_unsupported_streaming_guardrails( + pipeline: "GuardrailPipeline", translation: "BaseTranslation | None" +) -> tuple[str, ...]: return tuple( dict.fromkeys( - step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) + step.guardrail + for step in pipeline.steps + if not _pipeline_step_supports_streaming(step.guardrail, translation) ) ) -def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: - unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline) +def _pipeline_is_streamable( + policy_name: str, pipeline: "GuardrailPipeline", translation: "BaseTranslation | None" +) -> bool: + unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline, translation) if not unsupported: return True verbose_proxy_logger.warning( - "Policy '%s' has post_call pipeline guardrails with neither the unified apply_guardrail interface nor a " - "post-call hook, one of which streaming pipelines need; the stream skips the pipeline and its guardrails " - "run on their own: %s", + "Policy '%s' has post_call pipeline guardrails a streaming pipeline cannot run on this route yet; they " + "need the unified apply_guardrail interface, or a post-call hook without a streaming iterator hook on a " + "route whose translation assembles the streamed response. The stream skips the pipeline and its " + "guardrails run on their own: %s", policy_name, ", ".join(unsupported), ) return False -def _route_supports_streaming_pipelines(user_api_key_dict: UserAPIKeyAuth) -> bool: - return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None +def _streaming_pipeline_translation(user_api_key_dict: UserAPIKeyAuth) -> "BaseTranslation | None": + resolved: Final = resolve_endpoint_translation(user_api_key_dict, None) + return None if resolved is None else resolved[1] + + +def _route_supports_streaming_pipelines( + user_api_key_dict: UserAPIKeyAuth, translation: "BaseTranslation | None" +) -> bool: + return not user_api_key_dict.request_route or translation is not None def stream_gated_guardrail_names( request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth ) -> frozenset[str]: - if not _route_supports_streaming_pipelines(user_api_key_dict): + translation: Final = _streaming_pipeline_translation(user_api_key_dict) + if not _route_supports_streaming_pipelines(user_api_key_dict, translation): return frozenset() return _pipeline_step_guardrail_names( tuple( (policy_name, pipeline) for policy_name, pipeline in _post_call_pipelines(request_data) - if not _pipeline_unsupported_streaming_guardrails(pipeline) + if not _pipeline_unsupported_streaming_guardrails(pipeline, translation) ) ) @@ -589,8 +613,9 @@ def _streamable_post_call_pipelines( Streaming pipelines scan the buffered stream through the endpoint guardrail translation of the request route, so every step's guardrail needs either the - unified apply_guardrail interface or a post-call hook to run against the - assembled response, and the route needs a translation. A pipeline that + unified apply_guardrail interface or, on a route whose translation assembles + the streamed response, a post-call hook that is its only streaming path, and + the route needs a translation. A pipeline that cannot be run that way yet is left out and its guardrails run on the stream on their own, the way they did before pipelines ran on streams at all, with a warning naming the pipeline. @@ -598,7 +623,8 @@ def _streamable_post_call_pipelines( post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: return () - if not _route_supports_streaming_pipelines(user_api_key_dict): + translation: Final = _streaming_pipeline_translation(user_api_key_dict) + if not _route_supports_streaming_pipelines(user_api_key_dict, translation): verbose_proxy_logger.warning( "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet " "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run " @@ -610,7 +636,7 @@ def _streamable_post_call_pipelines( return tuple( (policy_name, pipeline) for policy_name, pipeline in post_call_pipelines - if _pipeline_is_streamable(policy_name, pipeline) + if _pipeline_is_streamable(policy_name, pipeline, translation) ) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 6de7f7b593d..e2523567065 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1589,28 +1589,14 @@ async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only assert chunks == [_tool_only_chunk()] -class _ResponselessLegacyScanningTranslation(_LegacyScanningTranslation): - """Like a handler that never stores the assembled response under request_data["response"].""" +class _NoHooksGuardrail(CustomGuardrail): + pass - def post_call_hook_response(self, response): - return response - async def process_output_streaming_response( - self, - responses_so_far, - guardrail_to_apply, - litellm_logging_obj=None, - user_api_key_dict=None, - request_data=None, - deliver_ended_stream_rewrites=False, - ): - await guardrail_to_apply.apply_guardrail( - inputs={"texts": [responses_so_far[0]["text"]]}, - request_data=request_data, - input_type="response", - logging_obj=litellm_logging_obj, - ) - return responses_so_far +class _IteratorAndLegacyHookGuardrail(_LegacyHookGuardrail): + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + async for item in response: + yield item class _UnscannableRewriteTranslation(_LegacyScanningTranslation): @@ -1623,21 +1609,11 @@ class _UnscannableRewriteTranslation(_LegacyScanningTranslation): return response -@pytest.mark.asyncio -async def test_streaming_step_leaves_the_stream_alone_when_the_hook_gets_no_response(monkeypatch, caplog): - guardrail = _LegacyHookGuardrail() - chunks = [_chunk()] - - result = await _run_legacy_streaming_step( - monkeypatch, guardrail, chunks, translation=_ResponselessLegacyScanningTranslation() - ) - - assert result.terminal_action == "allow" - assert [step.outcome for step in result.step_results] == ["pass"] - assert guardrail.calls[0]["response"] is None - assert chunks == [_chunk()] - assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] - assert not any("discarded" in record.getMessage() for record in caplog.records) +def test_streaming_execution_runs_legacy_hooks_only_when_that_hook_is_their_only_streaming_path(): + assert PipelineExecutor.supports_streaming_execution(_LegacyHookGuardrail()) is True + assert PipelineExecutor.supports_streaming_execution(_NativeHooksGuardrail()) is True + assert PipelineExecutor.supports_streaming_execution(_IteratorAndLegacyHookGuardrail()) is False + assert PipelineExecutor.supports_streaming_execution(_NoHooksGuardrail(guardrail_name="neither")) is False @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 46d6f2bb7a9..ac303e8a027 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -28,7 +28,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.integrations.prometheus import PrometheusLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header -from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines +from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines, stream_gated_guardrail_names from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks from litellm.types.proxy.policy_engine.pipeline_types import ( @@ -1539,6 +1539,21 @@ def _iterator_hook_only_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuar return IteratorHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) +def _iterator_and_legacy_hook_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuardrail: + class IteratorAndLegacyHookGuardrail(CustomGuardrail): + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + seen["success_hook_calls"] = seen.get("success_hook_calls", 0) + 1 + return None + + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + seen["iterator_hook_calls"] = seen.get("iterator_hook_calls", 0) + 1 + async for item in response: + item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" + yield item + + return IteratorAndLegacyHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) + + def _rewritten_model_response(response: Any) -> litellm.ModelResponse: payload = response.model_dump() payload["choices"][0]["message"]["content"] = "[REWRITTEN] " + payload["choices"][0]["message"]["content"] @@ -1572,6 +1587,42 @@ def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator assert not any("'governed'" in message or "gr-legacy" in message for message in _warnings(caplog)) +@pytest.mark.parametrize( + "request_route", + [None, "/v1/completions", "/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"], +) +def test_streamable_post_call_pipelines_keeps_legacy_hooks_off_routes_that_assemble_no_response( + make_user_api_key_auth, monkeypatch, caplog, request_route +): + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail({})]) + legacy = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + data = {"metadata": {"_guardrail_pipelines": [("legacy-governance", legacy)]}} + auth = make_user_api_key_auth(request_route=request_route) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + streamable = _streamable_post_call_pipelines(data, auth) + + assert streamable == () + assert stream_gated_guardrail_names(data, auth) == frozenset() + assert any("'legacy-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + + +def test_streamable_post_call_pipelines_keeps_guardrails_with_their_own_iterator_hook_on_their_own_path( + make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_iterator_and_legacy_hook_guardrail("gr-post", {})]) + both_hooks = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + data = {"metadata": {"_guardrail_pipelines": [("both-hooks", both_hooks)]}} + auth = make_user_api_key_auth(request_route="/v1/chat/completions") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + streamable = _streamable_post_call_pipelines(data, auth) + + assert streamable == () + assert stream_gated_guardrail_names(data, auth) == frozenset() + assert any("'both-hooks'" in message and "gr-post" in message for message in _warnings(caplog)) + + def test_streamable_post_call_pipelines_is_empty_on_route_without_translation( make_user_api_key_auth, monkeypatch, caplog ): @@ -1762,6 +1813,30 @@ async def test_streaming_iterator_hook_runs_iterator_hook_guardrail_whose_pipeli assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) +@pytest.mark.asyncio +async def test_streaming_iterator_hook_runs_the_iterator_hook_of_a_guardrail_that_also_has_a_post_call_hook( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + seen: Dict[str, Any] = {} + monkeypatch.setattr(litellm, "callbacks", [_iterator_and_legacy_hook_guardrail("gr-post", seen)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_stream_chunks()), + request_data=data, + ) + ] + + assert seen == {"iterator_hook_calls": 1} + assert [item.choices[0].delta.content for item in delivered] == ["[governed] hello ", "[governed] world"] + assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + + @pytest.mark.asyncio @pytest.mark.parametrize( "rewrite_attribute, value", From 9f21ae395aa298ab61331fb74e85d4769aae05e5 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 13:16:04 +0000 Subject: [PATCH 37/81] fix(registry): correct eu Claude 3.5 Haiku Bedrock pricing, add Nova v1 tool_choice, Azure gpt-5.5 snapshot retirement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 65 ++++++++++++------- model_prices_and_context_window.json | 65 ++++++++++++------- .../test_ai_policy_suggester.py | 2 +- tests/test_litellm/test_utils.py | 40 ++++++++---- 4 files changed, 111 insertions(+), 61 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43def22ba58..d1f1a2d2b48 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -364,7 +364,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -537,7 +538,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -551,7 +553,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-sonic-v1:0": { "deprecation_date": "2026-09-14", @@ -2876,7 +2879,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -2888,7 +2892,8 @@ "output_cost_per_token": 1.48e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "apac.amazon.nova-pro-v1:0": { "input_cost_per_token": 8.4e-07, @@ -2902,7 +2907,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "deprecation_date": "2026-07-30", @@ -8064,7 +8070,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8108,7 +8115,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8152,7 +8160,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -12317,7 +12326,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12496,7 +12506,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "input_cost_per_token": 4.2e-08, @@ -12508,7 +12519,8 @@ "output_cost_per_token": 1.68e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, @@ -12522,7 +12534,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -20990,7 +21003,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -21002,7 +21016,8 @@ "output_cost_per_token": 1.84e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "eu.amazon.nova-pro-v1:0": { "input_cost_per_token": 1.05e-06, @@ -21017,24 +21032,25 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { - "input_cost_per_token": 2.5e-07, + "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.25e-06, + "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07, + "cache_read_input_token_cost": 8e-08, + "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -43795,7 +43811,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -43807,7 +43824,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "us.amazon.nova-premier-v1:0": { "deprecation_date": "2026-09-14", @@ -43836,7 +43854,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43def22ba58..d1f1a2d2b48 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -364,7 +364,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -537,7 +538,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -551,7 +553,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-sonic-v1:0": { "deprecation_date": "2026-09-14", @@ -2876,7 +2879,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -2888,7 +2892,8 @@ "output_cost_per_token": 1.48e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "apac.amazon.nova-pro-v1:0": { "input_cost_per_token": 8.4e-07, @@ -2902,7 +2907,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "deprecation_date": "2026-07-30", @@ -8064,7 +8070,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8108,7 +8115,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8152,7 +8160,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -12317,7 +12326,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12496,7 +12506,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "input_cost_per_token": 4.2e-08, @@ -12508,7 +12519,8 @@ "output_cost_per_token": 1.68e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, @@ -12522,7 +12534,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -20990,7 +21003,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -21002,7 +21016,8 @@ "output_cost_per_token": 1.84e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "eu.amazon.nova-pro-v1:0": { "input_cost_per_token": 1.05e-06, @@ -21017,24 +21032,25 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { - "input_cost_per_token": 2.5e-07, + "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.25e-06, + "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07, + "cache_read_input_token_cost": 8e-08, + "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -43795,7 +43811,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -43807,7 +43824,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "us.amazon.nova-premier-v1:0": { "deprecation_date": "2026-09-14", @@ -43836,7 +43854,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, diff --git a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py index 93dc429168f..bb71d67f24e 100644 --- a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py +++ b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py @@ -265,7 +265,7 @@ class TestSuggesterRejectsModelsWithoutToolCalling: def test_a_model_without_forced_tool_choice_support_remains_eligible(self, local_model_cost_map): supported_params = litellm.get_supported_openai_params( - model="amazon.nova-pro-v1:0", + model="meta.llama4-scout-17b-instruct-v1:0", custom_llm_provider="bedrock", ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d10e7634720..ec3a141a490 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1409,23 +1409,35 @@ def test_supports_tool_choice_simple_tests(): is True ) - assert ( - litellm.utils.supports_tool_choice(model="us.amazon.nova-micro-v1:0") is False - ) - assert ( - litellm.utils.supports_tool_choice(model="bedrock/us.amazon.nova-micro-v1:0") - is False - ) - assert ( - litellm.utils.supports_tool_choice( - model="us.amazon.nova-micro-v1:0", custom_llm_provider="bedrock_converse" - ) - is False - ) - assert litellm.utils.supports_tool_choice(model="perplexity/sonar") is False +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model", + [ + "amazon.nova-lite-v1:0", + "amazon.nova-micro-v1:0", + "amazon.nova-pro-v1:0", + "apac.amazon.nova-lite-v1:0", + "apac.amazon.nova-micro-v1:0", + "apac.amazon.nova-pro-v1:0", + "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", + "bedrock/us-gov-west-1/amazon.nova-lite-v1:0", + "bedrock/us-gov-west-1/amazon.nova-micro-v1:0", + "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", + "eu.amazon.nova-lite-v1:0", + "eu.amazon.nova-micro-v1:0", + "eu.amazon.nova-pro-v1:0", + "us.amazon.nova-lite-v1:0", + "us.amazon.nova-micro-v1:0", + "us.amazon.nova-pro-v1:0", + ], +) +def test_amazon_nova_v1_understanding_models_support_tool_choice(model: str) -> None: + assert litellm.utils.supports_tool_choice(model=model) is True + + def test_check_provider_match(): """ Test the _check_provider_match function for various provider scenarios From b067729082783e02c13a7b757360caa09d1297be Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 13:51:40 +0000 Subject: [PATCH 38/81] fix(registry): add xAI Imagine Video 720p per second rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ model_prices_and_context_window.schema.json | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d1f1a2d2b48..dc720e2629a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59849,6 +59849,7 @@ "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, "source": "https://docs.x.ai/docs/models/grok-imagine-video", "supported_modalities": [ "text", @@ -59866,6 +59867,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59883,6 +59885,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59900,6 +59903,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d1f1a2d2b48..dc720e2629a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59849,6 +59849,7 @@ "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, "source": "https://docs.x.ai/docs/models/grok-imagine-video", "supported_modalities": [ "text", @@ -59866,6 +59867,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59883,6 +59885,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59900,6 +59903,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 47a1934a703..7ed1e7e568b 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -478,6 +478,10 @@ "type": "number", "minimum": 0 }, + "output_cost_per_second_720p": { + "type": "number", + "minimum": 0 + }, "output_cost_per_token": { "type": "number", "minimum": 0, From 2881b8cd4553f1dab28e7220e7f5a1f30e960aed Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 14:09:25 +0000 Subject: [PATCH 39/81] fix(cost): carry output_cost_per_second_720p through model info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/router.py | 1 + litellm/types/utils.py | 2 ++ litellm/utils.py | 1 + tests/test_litellm/test_utils.py | 2 ++ tests/test_litellm/test_video_generation.py | 26 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +++ 6 files changed, 36 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 5c9eab30f3d..6b707a544a2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -525,6 +525,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): input_cost_per_second: float | None output_cost_per_second: float | None output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_720p: ReadOnly[float | None] output_cost_per_second_1080p: float | None output_cost_per_second_4k: ReadOnly[float | None] num_retries: int | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d62f00f3676..ab0cc5f959c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -318,6 +318,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_720p: ReadOnly[float | None] output_cost_per_second_4k: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_credit: float | None # for OCR models priced by credit @@ -3522,6 +3523,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None output_cost_per_second_480p: float | None = None + output_cost_per_second_720p: float | None = None output_cost_per_second_4k: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 36b48d3b8d8..d0ff4917616 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5913,6 +5913,7 @@ def _get_model_info_helper( output_cost_per_second=_model_info.get("output_cost_per_second", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), + output_cost_per_second_720p=_model_info.get("output_cost_per_second_720p", None), output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ec3a141a490..357c0d1c363 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -814,6 +814,7 @@ def validate_model_cost_values(model_data, exceptions=None): "input_cost_per_second", "output_cost_per_second", "output_cost_per_second_480p", + "output_cost_per_second_720p", "output_cost_per_second_1080p", "output_cost_per_second_4k", "input_cost_per_query", @@ -1037,6 +1038,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_pixel": {"type": "number"}, "output_cost_per_second": {"type": "number"}, "output_cost_per_second_480p": {"type": "number"}, + "output_cost_per_second_720p": {"type": "number"}, "output_cost_per_second_1080p": {"type": "number"}, "output_cost_per_second_4k": {"type": "number"}, "output_cost_per_token": {"type": "number"}, diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 2a60ff9c4b5..f3cd4618078 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -532,6 +532,32 @@ class TestVideoGeneration: assert abs(cost_for("runwayml/seedance2_5", "480p", 8.0) - 1.6) < 0.001 assert abs(cost_for("runwayml/gen4.5", None, 8.0) - 0.96) < 0.001 + def test_completion_cost_xai_imagine_video_720p_tier_from_cost_map(self, monkeypatch): + """720p xAI Imagine Video requests bill the published 720p rate, not the 480p base rate.""" + from litellm.cost_calculator import completion_cost + + local_map_path = os.path.join( + os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" + ) + with open(local_map_path, "r") as f: + monkeypatch.setattr(litellm, "model_cost", json.load(f)) + + def cost_for(model: str, resolution: str, duration: float) -> float: + mock_response = MagicMock() + mock_response.usage = {"duration_seconds": duration, "video_resolution": resolution} + type(mock_response)._hidden_params = {} + return completion_cost( + completion_response=mock_response, + model=model, + call_type="create_video", + custom_llm_provider="xai", + ) + + assert abs(cost_for("xai/grok-imagine-video", "720p", 10.0) - 0.7) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "720p", 10.0) - 1.4) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "480p", 10.0) - 0.8) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "1080p", 10.0) - 2.5) < 0.001 + def test_completion_cost_veo_31_tiers_pin_published_rates(self, monkeypatch): """The gemini and vertex_ai veo 3.1 entries bill Google's published per-second tier rates.""" from litellm.cost_calculator import completion_cost diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 83b0d58f2b2..ce50978602f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29629,6 +29629,8 @@ export interface components { output_cost_per_second_480p?: number | null; /** Output Cost Per Second 4K */ output_cost_per_second_4k?: number | null; + /** Output Cost Per Second 720P */ + output_cost_per_second_720p?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ @@ -39820,6 +39822,8 @@ export interface components { output_cost_per_second_480p?: number | null; /** Output Cost Per Second 4K */ output_cost_per_second_4k?: number | null; + /** Output Cost Per Second 720P */ + output_cost_per_second_720p?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ From 6dbaf43ba2951e403fd1efe6920eaf6c30550c42 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 16:42:07 +0000 Subject: [PATCH 40/81] fix(registry): drop OpenAI shutdown date from shared computer-use-preview entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dc720e2629a..cdd9fc7b6d4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14560,7 +14560,6 @@ "supports_tool_choice": true }, "computer-use-preview": { - "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dc720e2629a..cdd9fc7b6d4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14560,7 +14560,6 @@ "supports_tool_choice": true }, "computer-use-preview": { - "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, From 00e4380cb5b84b9681cbf6e14886cdbaf328cf4b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 19:15:44 +0000 Subject: [PATCH 41/81] fix(registry): add Bedrock gpt-6-astra CRIS + mantle profiles, embed-v4/pegasus global profiles, gpt-image-2.5 entries; cap Vertex grok-4.1-fast output at 128k Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 188 +++++++++++++++++- model_prices_and_context_window.json | 188 +++++++++++++++++- 2 files changed, 368 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cdd9fc7b6d4..664c2737157 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -759,6 +759,14 @@ "mode": "chat", "supports_video_input": true }, + "global.twelvelabs.pegasus-1-2-v1:0": { + "input_cost_per_video_per_second": 0.00049, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_video_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "amazon.titan-text-express-v1": { "input_cost_per_token": 1.3e-06, "litellm_provider": "bedrock", @@ -14431,6 +14439,28 @@ "output_vector_size": 1536, "supports_embedding_image_input": true }, + "us.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "cohere/embed-v4.0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", @@ -29748,6 +29778,66 @@ "supports_vision": true, "supports_pdf_input": true }, + "gpt-image-2.5-flare": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-flare-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, @@ -48040,8 +48130,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -48056,8 +48146,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -55429,6 +55519,96 @@ "supports_reasoning": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "us.openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "global.openai.gpt-6-astra": { + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cdd9fc7b6d4..664c2737157 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -759,6 +759,14 @@ "mode": "chat", "supports_video_input": true }, + "global.twelvelabs.pegasus-1-2-v1:0": { + "input_cost_per_video_per_second": 0.00049, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_video_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "amazon.titan-text-express-v1": { "input_cost_per_token": 1.3e-06, "litellm_provider": "bedrock", @@ -14431,6 +14439,28 @@ "output_vector_size": 1536, "supports_embedding_image_input": true }, + "us.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "cohere/embed-v4.0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", @@ -29748,6 +29778,66 @@ "supports_vision": true, "supports_pdf_input": true }, + "gpt-image-2.5-flare": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-flare-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, @@ -48040,8 +48130,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -48056,8 +48146,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -55429,6 +55519,96 @@ "supports_reasoning": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "us.openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "global.openai.gpt-6-astra": { + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, From 35def27e7b145c7aef7463bef34db4e56a2e8d9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:37:29 -0700 Subject: [PATCH 42/81] fix(convert_dict_to_response): accept only a real list as choices and keep /v1/messages alive on an empty one Narrows the no-choices guard so a dict, string, or None still raises the APIError while an empty list passes through, guards the non-stream Anthropic bridge against indexing an empty choices list, and repairs test_completion_missing_role, whose raw-response mock was patched in as the create() callable itself so the handler only ever saw a MagicMock --- .../convert_dict_to_response.py | 8 ++--- .../adapters/transformation.py | 3 +- .../test_convert_dict_to_response.py | 30 +++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 15 ++++++++++ tests/test_litellm/test_main.py | 2 +- 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 7d8072ba622..29b75812bfb 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,7 +3,7 @@ import json import re import time import traceback -from collections.abc import Iterable, Sequence +from collections.abc import Sequence from typing import Final, Literal, cast import litellm @@ -179,7 +179,7 @@ async def convert_to_streaming_response_async( choice_list: Final[list[StreamingChoices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( @@ -287,7 +287,7 @@ def convert_to_streaming_response( model_response_object: Final = ModelResponseStream() choice_list: Final[list[StreamingChoices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( @@ -623,7 +623,7 @@ def convert_to_model_response_object( return convert_to_streaming_response(response_object=response_object) choice_list: Final[list[Choices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index db890662132..4bce760e943 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1487,8 +1487,9 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason + openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop" translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( - openai_finish_reason=response.choices[0].finish_reason + openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( "refusal" diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 5406df691aa..ea50bff462e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,3 +1,5 @@ +from typing import Final + import pytest from litellm.constants import RESPONSE_FORMAT_TOOL_NAME @@ -167,3 +169,31 @@ def test_convert_missing_choices_raises_api_error() -> None: ) assert "no 'choices'" in str(exc_info.value) + +@pytest.mark.parametrize("choices", [{}, "", None, 0]) +@pytest.mark.asyncio +async def test_convert_non_list_choices_raises_api_error(choices: object) -> None: + from litellm.exceptions import APIError + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + convert_to_streaming_response_async, + ) + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": choices, + } + with pytest.raises(APIError, match="no 'choices'"): + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + with pytest.raises(APIError, match="no 'choices'"): + list(convert_to_streaming_response(response_object=resp)) + with pytest.raises(APIError, match="no 'choices'"): + async for _ in convert_to_streaming_response_async(response_object=resp): + pass diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index c59ec70b015..00b3a0633f2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -41,6 +41,21 @@ from litellm.types.utils import ( ) +def test_translate_openai_response_to_anthropic_empty_choices() -> None: + response: Final = ModelResponse( + id="chatcmpl-empty", + model="gemini-3.5-flash", + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10), + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + assert result["usage"]["input_tokens"] == 10 + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 038df3656fe..de6322f27ee 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -120,7 +120,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") with patch.object( - client.chat.completions.with_raw_response, "create", mock_raw_response + client.chat.completions.with_raw_response, "create", MagicMock(return_value=mock_raw_response) ) as mock_create: litellm.completion( model="gpt-4o-mini", From 261777d633c6c5466a0711a7a889ca2cfb0a08d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:40:29 -0700 Subject: [PATCH 43/81] fix(databricks): keep top-level reasoning_content from OpenAI-compatible gateway models The Databricks chat transformation only parsed reasoning out of FMAPI-style reasoning content blocks, so external models behind Databricks AI Gateway that return the OpenAI-style top-level reasoning_content string lost it, both in the final message and in every streamed delta. Fall back to the shared OpenAI reasoning helper when no reasoning block exists, and keep the delta's own reasoning_content when streaming. --- .../llms/databricks/chat/transformation.py | 33 ++++++-- litellm/types/llms/databricks.py | 9 ++ .../test_databricks_chat_transformation.py | 83 +++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index e2e2ea5b553..54c7040daf9 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo _should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, # pyright: ignore[reportPrivateUsage] # same import as the OpenAI transformation strip_litellm_internal_message_fields, strip_name_from_message, ) @@ -23,7 +24,9 @@ from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.databricks import ( AllDatabricksContentValues, DatabricksChoice, + DatabricksDelta, DatabricksFunction, + DatabricksMessage, DatabricksResponse, DatabricksTool, ) @@ -534,6 +537,19 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): thinking_blocks.append(thinking_block) return reasoning_content, thinking_blocks + @staticmethod + def extract_top_level_reasoning_content(delta: DatabricksDelta) -> str | None: + return delta.get("reasoning_content") + + @staticmethod + def resolve_reasoning_and_content( + message: DatabricksMessage, block_reasoning_content: str | None + ) -> tuple[str | None, str | None]: + content_str: Final = DatabricksConfig.extract_content_str(message["content"]) + if block_reasoning_content is not None: + return block_reasoning_content, content_str + return _extract_reasoning_content({**message, "content": content_str}) + @staticmethod def extract_citations( content: AllDatabricksContentValues | None, @@ -577,14 +593,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): finish_reason = "stop" if translated_message is None: - ## get the content str - content_str = DatabricksConfig.extract_content_str(choice["message"]["content"]) - - ## get the reasoning content ( - reasoning_content, + block_reasoning_content, thinking_blocks, ) = DatabricksConfig.extract_reasoning_content(choice["message"].get("content")) + reasoning_content, content_str = DatabricksConfig.resolve_reasoning_and_content( + choice["message"], block_reasoning_content + ) citations = DatabricksConfig.extract_citations(choice["message"].get("content")) @@ -738,12 +753,16 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): # extract the reasoning content ( - reasoning_content, + block_reasoning_content, thinking_blocks, ) = DatabricksConfig.extract_reasoning_content(choice["delta"].get("content")) choice["delta"]["content"] = content_str - choice["delta"]["reasoning_content"] = reasoning_content + choice["delta"]["reasoning_content"] = ( + block_reasoning_content + if block_reasoning_content is not None + else DatabricksConfig.extract_top_level_reasoning_content(choice["delta"]) + ) choice["delta"]["thinking_blocks"] = thinking_blocks translated_choices.append(choice) return ModelResponseStream( diff --git a/litellm/types/llms/databricks.py b/litellm/types/llms/databricks.py index e87a684aab8..a9c027bd2de 100644 --- a/litellm/types/llms/databricks.py +++ b/litellm/types/llms/databricks.py @@ -2,6 +2,7 @@ from typing import Any, Literal from pydantic import BaseModel from typing_extensions import ( + ReadOnly, Required, TypedDict, ) @@ -57,6 +58,14 @@ class DatabricksMessage(TypedDict, total=False): role: Required[str] content: Required[AllDatabricksContentValues] tool_calls: list[DatabricksTool] | None + reasoning_content: ReadOnly[str | None] + reasoning: ReadOnly[str | None] + + +class DatabricksDelta(TypedDict, total=False): + role: ReadOnly[str] + content: ReadOnly[AllDatabricksContentValues | None] + reasoning_content: ReadOnly[str | None] class DatabricksChoice(TypedDict, total=False): diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 02655fb7f77..89d1aa52838 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -590,3 +590,86 @@ def test_chunk_parser_without_usage_still_parses_content(): assert result.id == "chatcmpl-test" assert result.model == "databricks-claude-sonnet-5" assert result.choices[0]["delta"]["content"] == "hi" + + +@pytest.mark.parametrize("reasoning_key", ["reasoning_content", "reasoning"]) +def test_transform_choices_surfaces_top_level_reasoning_content(reasoning_key: str) -> None: + config = DatabricksConfig() + databricks_choices = [ + { + "message": { + "role": "assistant", + "content": "391", + reasoning_key: "We need answer just number. 17*23=391.", + }, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.content == "391" + assert choices[0].message.reasoning_content == "We need answer just number. 17*23=391." + assert getattr(choices[0].message, "thinking_blocks", None) is None + + +def test_transform_choices_parses_think_tags_in_string_content(): + config = DatabricksConfig() + databricks_choices = [ + { + "message": {"role": "assistant", "content": "17 times 23391"}, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.content == "391" + assert choices[0].message.reasoning_content == "17 times 23" + + +def test_transform_choices_prefers_reasoning_blocks_over_top_level_field(): + config = DatabricksConfig() + databricks_choices = [ + { + "message": { + "role": "assistant", + "content": [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "from block"}]}, + {"type": "text", "text": "391"}, + ], + "reasoning_content": "from field", + }, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.reasoning_content == "from block" + assert choices[0].message.content == "391" + + +def test_chunk_parser_surfaces_top_level_reasoning_delta(): + iterator = DatabricksChatResponseIterator(None, sync_stream=True) + chunk = { + "id": "1", + "object": "chat.completion.chunk", + "created": 0, + "model": "lit-qa-deepseek-v4-flash", + "choices": [ + { + "delta": {"role": "assistant", "content": "", "reasoning_content": "We need answer"}, + "index": 0, + "finish_reason": None, + } + ], + } + + parsed = iterator.chunk_parser(chunk) + + assert parsed.choices[0].delta.reasoning_content == "We need answer" + assert parsed.choices[0].delta.content == "" From 420282acb7704772ffb47c2ed4f84d455a2a385d Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 19:55:02 +0000 Subject: [PATCH 44/81] fix(registry): set max_output_tokens on vertex_ai/xai/grok-4.3 and grok-4.6 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 664c2737157..885021692c9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48204,6 +48204,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 200000, + "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -48222,6 +48223,7 @@ "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 524288, + "max_output_tokens": 524288, "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 664c2737157..885021692c9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48204,6 +48204,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 200000, + "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -48222,6 +48223,7 @@ "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 524288, + "max_output_tokens": 524288, "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, From 17ca562b6a5d208d1b41a58933491936d4cefee2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:01:11 -0700 Subject: [PATCH 45/81] test(llm_translation): expect an empty choices list to convert instead of raising --- .../test_convert_dict_to_chat_completion.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index b6e30ddc711..09d1adba17c 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1623,15 +1623,11 @@ class TestMissingChoicesGuard: assert "no 'choices'" in exc_info.value.message - def test_convert_to_model_response_object_empty_choices_raises_api_error(self): - """Empty choices list raises APIError, same as missing/null choices. + def test_convert_to_model_response_object_empty_choices_returns_empty_list(self): + """An empty choices list is a real provider answer, so it converts to choices=[] instead of raising. - Provider-specific repair (e.g. github_copilot synthesizing choices for - Anthropic-native responses) happens before this guard, in the provider - config; the core utility keeps treating empty choices as an error. + See: https://github.com/BerriAI/litellm/issues/40276 """ - from litellm.exceptions import APIError - response_object = { "id": "msg_123", "model": "some-model", @@ -1639,13 +1635,14 @@ class TestMissingChoicesGuard: "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, } - with pytest.raises(APIError) as exc_info: - convert_to_model_response_object( - response_object=response_object, - model_response_object=ModelResponse(), - ) + result = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) - assert "no 'choices'" in exc_info.value.message + assert isinstance(result, ModelResponse) + assert result.choices == [] + assert result.usage.prompt_tokens == 10 def test_convert_to_model_response_object_null_choices_raises_api_error(self): """choices=None raises APIError.""" From f3a2844080f5b2dd003a084eece2861e78a1870d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:05:11 -0700 Subject: [PATCH 46/81] test(convert_dict_to_response): keep the regression test locals final and comment-free --- .../test_convert_dict_to_response.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index ea50bff462e..c3e99f01cec 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -108,7 +108,7 @@ def test_convert_empty_choices_response() -> None: convert_to_streaming_response, ) - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", @@ -117,7 +117,7 @@ def test_convert_empty_choices_response() -> None: "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, "vertex_ai_safety_results": ["blocked"], } - result = convert_to_model_response_object( + result: Final = convert_to_model_response_object( response_object=resp, model_response_object=ModelResponse(), response_type="completion", @@ -125,8 +125,7 @@ def test_convert_empty_choices_response() -> None: assert result.choices == [] assert getattr(result, "vertex_ai_safety_results") == ["blocked"] - # Test sync streaming generator handles empty choices - sync_stream = list(convert_to_streaming_response(response_object=resp)) + sync_stream: Final = list(convert_to_streaming_response(response_object=resp)) assert len(sync_stream) == 1 assert sync_stream[0].choices == [] @@ -137,7 +136,7 @@ async def test_convert_empty_choices_response_async() -> None: convert_to_streaming_response_async, ) - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", @@ -145,9 +144,7 @@ async def test_convert_empty_choices_response_async() -> None: "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, } - async_chunks = [] - async for chunk in convert_to_streaming_response_async(response_object=resp): - async_chunks.append(chunk) + async_chunks: Final = [chunk async for chunk in convert_to_streaming_response_async(response_object=resp)] assert len(async_chunks) == 1 assert async_chunks[0].choices == [] @@ -155,7 +152,7 @@ async def test_convert_empty_choices_response_async() -> None: def test_convert_missing_choices_raises_api_error() -> None: from litellm.exceptions import APIError - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", From fe242f87028a9bd8ae2fcd4e579436f32614c116 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:16:57 -0700 Subject: [PATCH 47/81] test(databricks): pin the streamed reasoning delta shape and alias --- .../databricks/chat/test_databricks_chat_transformation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 89d1aa52838..38d5844b9b9 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -653,7 +653,8 @@ def test_transform_choices_prefers_reasoning_blocks_over_top_level_field(): assert choices[0].message.content == "391" -def test_chunk_parser_surfaces_top_level_reasoning_delta(): +@pytest.mark.parametrize("reasoning_key", ["reasoning_content", "reasoning"]) +def test_chunk_parser_surfaces_top_level_reasoning_delta(reasoning_key: str) -> None: iterator = DatabricksChatResponseIterator(None, sync_stream=True) chunk = { "id": "1", @@ -662,7 +663,7 @@ def test_chunk_parser_surfaces_top_level_reasoning_delta(): "model": "lit-qa-deepseek-v4-flash", "choices": [ { - "delta": {"role": "assistant", "content": "", "reasoning_content": "We need answer"}, + "delta": {"role": "assistant", "content": None, reasoning_key: "We need answer"}, "index": 0, "finish_reason": None, } @@ -672,4 +673,4 @@ def test_chunk_parser_surfaces_top_level_reasoning_delta(): parsed = iterator.chunk_parser(chunk) assert parsed.choices[0].delta.reasoning_content == "We need answer" - assert parsed.choices[0].delta.content == "" + assert parsed.choices[0].delta.content is None From a9cce4f1ed15af0082a66233732d8e27f95b6bc8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:38:51 -0700 Subject: [PATCH 48/81] fix(guardrails): scan and rewrite Responses custom_tool_call output items Post-call guardrails on /v1/responses only treated function_call output items as tool calls, so a custom_tool_call item (Codex's exec shell tool on GPT-5.6 models) was never scanned or masked, non-streaming and streaming alike. Both item types now flow through the shared tool_call_dict_from_output_item helper, ended-stream delivery syncs the custom_tool_call_input delta/done events and the item's input field, and the completed-response scan key fingerprints both kinds of item. Non-streaming Responses tool-call MASK rewrites were also never written back to the output item even for function_call; they are now. --- .../transformation.py | 8 +- .../guardrail_translation/handler.py | 157 +++++++----- ...test_openai_responses_guardrail_handler.py | 229 +++++++++++++++++- 3 files changed, 322 insertions(+), 72 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4fe069b0b7d..aa8337b6300 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -227,7 +227,7 @@ class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False): provider_specific_fields: Mapping[str, object] -def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict: +def tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict: """Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw string payload in ``input`` rather than ``arguments``; both map to @@ -755,7 +755,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Tool calls accumulate into the single trailing tool_calls choice # like the typed branches above; a choice per call would hide every # call after choices[0] from chat clients - accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item, tool_call_index)) + accumulated_tool_calls.append(tool_call_dict_from_output_item(raw_item, tool_call_index)) tool_call_index += 1 elif handle_raw_dict_callback is not None: choice, index = handle_raw_dict_callback(item=raw_item, index=index) @@ -1409,7 +1409,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") in ("function_call", "custom_tool_call"): - converted: Final = _tool_call_dict_from_output_item(output_item, parsed_chunk.get("output_index", 0)) + converted: Final = tool_call_dict_from_output_item(output_item, parsed_chunk.get("output_index", 0)) provider_specific_fields: Final = converted.get("provider_specific_fields") function_chunk: Final = ChatCompletionToolCallFunctionChunk( @@ -1484,7 +1484,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): index=0, delta=Delta( tool_calls=( - _tool_call_dict_from_output_item( + tool_call_dict_from_output_item( output_item, parsed_chunk.get("output_index", 0) ), ) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 280a8670c36..e78e1f56915 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -37,7 +37,6 @@ from itertools import accumulate, chain, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast -from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -45,6 +44,7 @@ from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, OpenAiResponsesToChatCompletionStreamIterator, + tool_call_dict_from_output_item, ) from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -84,7 +84,6 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import ( GenericResponseOutputItem, - OutputFunctionToolCall, OutputText, ) from litellm.types.utils import GenericGuardrailAPIInputs @@ -140,8 +139,18 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( ) -_FUNCTION_CALL_ARGUMENT_EVENT_TYPES: Final = frozenset( - {"response.function_call_arguments.delta", "response.function_call_arguments.done"} +_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) +_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"function_call": "arguments", "custom_tool_call": "input"} +) +_TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES: Final = frozenset( + {"response.function_call_arguments.delta", "response.custom_tool_call_input.delta"} +) +_TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"response.function_call_arguments.done": "arguments", "response.custom_tool_call_input.done": "input"} +) +_TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | frozenset( + _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS ) _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( @@ -180,8 +189,20 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp return {**item, field: converted_value} # mutable-ok: request input items must stay JSON-plain dicts -def _is_function_call_item(item: object) -> bool: - return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call") +def _is_tool_call_item(item: object) -> bool: + return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES + + +def _tool_call_output_item_mapping(item: object) -> Mapping[str, Any] | None: + if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES: + return None + if isinstance(item, Mapping): + return cast("Mapping[str, Any]", item) # cast-ok: output items are str-keyed JSON objects + return item.model_dump() if isinstance(item, BaseModel) else None + + +def _is_tool_call_output_item(item: object) -> bool: + return _tool_call_output_item_mapping(item) is not None def _last_message_role(messages: Sequence[object]) -> str | None: @@ -205,7 +226,7 @@ def _provenance_unit_bounds( start_indexes: Final = tuple( index for index in range(len(raw_input)) - if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") + if index == 0 or not (_is_tool_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") ) return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input)))) @@ -603,7 +624,7 @@ class OpenAIResponsesHandler(BaseTranslation): - response.output is a list of output items - Each output item can be: * GenericResponseOutputItem with a content list of OutputText objects - * ResponseFunctionToolCall with tool call data + * ResponseFunctionToolCall or CustomToolCallOutputItem with tool call data - Each OutputText object has a text field """ @@ -668,6 +689,7 @@ class OpenAIResponsesHandler(BaseTranslation): if response_model: inputs["model"] = response_model + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=request_data, @@ -676,6 +698,7 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( @@ -683,6 +706,15 @@ class OpenAIResponsesHandler(BaseTranslation): responses=guardrailed_texts, task_mappings=task_mappings, ) + self._write_tool_call_rewrites_to_output( + tool_call_items=tuple(item for item in response_output if _is_tool_call_output_item(item)), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=_tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) + else tool_calls_to_check + ), + ) verbose_proxy_logger.debug("OpenAI Responses API: Processed output response: %s", response) @@ -933,11 +965,12 @@ class OpenAIResponsesHandler(BaseTranslation): guardrail_name: str, ) -> None: """Write ended-stream guardrail tool-call rewrites into the completed - envelope's ``function_call`` items and sync the earlier stream events, - keyed by ``call_id``. The guardrail sees the envelope's function calls - in output order, which is how a rewritten call finds its ``call_id``; - the stream events find their call through the ``call_id`` on - ``output_item`` events and the ``item_id`` on argument events, since an + envelope's ``function_call`` and ``custom_tool_call`` items and sync the + earlier stream events, keyed by ``call_id``. The guardrail sees the + envelope's tool calls in output order, which is how a rewritten call + finds its ``call_id``; the stream events find their call through the + ``call_id`` on ``output_item`` events and the ``item_id`` on argument + and custom-input events, since an event's ``output_index`` need not match the envelope's (the chat bridge numbers tool calls from 1 while the envelope lists them after the message). A rewrite whose calls do not line up with the envelope, or @@ -945,18 +978,16 @@ class OpenAIResponsesHandler(BaseTranslation): pipeline executor discards it and releases the original events.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - function_call_items: Final = tuple( - output_item for output_item in outputs if stream_item_field(output_item, "type") == "function_call" - ) + tool_call_items: Final = tuple(output_item for output_item in outputs if _is_tool_call_output_item(output_item)) call_ids: Final = tuple( call_id - for output_item in function_call_items + for output_item in tool_call_items if isinstance(call_id := stream_item_field(output_item, "call_id"), str) and call_id ) stream_events: Final = responses_so_far[:-1] - call_id_by_item_id: Final = self._function_call_ids_by_item_id(stream_events) + call_id_by_item_id: Final = self._tool_call_ids_by_item_id(stream_events) event_call_ids: Final = tuple( - self._function_call_event_call_id(event, call_id_by_item_id) for event in stream_events + self._tool_call_event_call_id(event, call_id_by_item_id) for event in stream_events ) rewrites_by_call_id: Final = MappingProxyType( { @@ -966,11 +997,11 @@ class OpenAIResponsesHandler(BaseTranslation): } ) unresolved_argument_event: Final = any( - call_id is None and stream_item_field(event, "type") in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES + call_id is None and stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES for event, call_id in zip(stream_events, event_call_ids) ) if ( - len(call_ids) != len(function_call_items) + len(call_ids) != len(tool_call_items) or len(frozenset(call_ids)) != len(call_ids) or len(call_ids) != len(post_guardrail_tool_calls) or unresolved_argument_event @@ -981,10 +1012,10 @@ class OpenAIResponsesHandler(BaseTranslation): raise UndeliverableStreamRewrite(guardrail_name) for output_item, rewrite in ( (output_item, rewrites_by_call_id[call_id]) - for output_item, call_id in zip(function_call_items, call_ids) + for output_item, call_id in zip(tool_call_items, call_ids) if call_id in rewrites_by_call_id ): - self._write_function_call_item(output_item, rewrite.name, rewrite.arguments) + self._write_tool_call_item(output_item, rewrite.name, rewrite.arguments) delta_replacements: Final = MappingProxyType( {call_id: chain((rewrite.arguments,), repeat("")) for call_id, rewrite in rewrites_by_call_id.items()} ) @@ -992,16 +1023,18 @@ class OpenAIResponsesHandler(BaseTranslation): if call_id not in rewrites_by_call_id: continue match stream_item_field(event, "type"): - case "response.function_call_arguments.delta": + case str() as event_type if event_type in _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES: self._write_event_field(event, "delta", next(delta_replacements[call_id])) - case "response.function_call_arguments.done": - self._write_event_field(event, "arguments", rewrites_by_call_id[call_id].arguments) + case str() as event_type if event_type in _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS: + self._write_event_field( + event, _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS[event_type], rewrites_by_call_id[call_id].arguments + ) case "response.output_item.added": - self._write_function_call_item( + self._write_tool_call_item( stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, None ) case "response.output_item.done": - self._write_function_call_item( + self._write_tool_call_item( stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, rewrites_by_call_id[call_id].arguments, @@ -1009,8 +1042,23 @@ class OpenAIResponsesHandler(BaseTranslation): case _: pass + def _write_tool_call_rewrites_to_output( + self, + tool_call_items: Sequence[object], + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + ) -> None: + if len(tool_call_items) != len(post_guardrail_tool_calls): + return + for output_item, after in ( + (output_item, after) + for output_item, before, after in zip(tool_call_items, pre_guardrail_tool_calls, post_guardrail_tool_calls) + if after != before + ): + self._write_tool_call_item(output_item, after.name, after.arguments) + @staticmethod - def _function_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: + def _tool_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: items: Final = tuple( stream_item_field(event, "item") for event in stream_events @@ -1020,32 +1068,35 @@ class OpenAIResponsesHandler(BaseTranslation): { item_id: call_id for item in items - if stream_item_field(item, "type") == "function_call" + if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES and isinstance(item_id := stream_item_field(item, "id"), str) and isinstance(call_id := stream_item_field(item, "call_id"), str) } ) @staticmethod - def _function_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: + def _tool_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: event_type: Final = stream_item_field(event, "type") - if event_type in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES: + if event_type in _TOOL_CALL_PAYLOAD_EVENT_TYPES: item_id: Final = stream_item_field(event, "item_id") return call_id_by_item_id.get(item_id) if isinstance(item_id, str) else None if event_type not in _OUTPUT_ITEM_EVENT_TYPES: return None item: Final = stream_item_field(event, "item") call_id: Final = stream_item_field(item, "call_id") - return call_id if stream_item_field(item, "type") == "function_call" and isinstance(call_id, str) else None + return ( + call_id if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES and isinstance(call_id, str) else None + ) @staticmethod - def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: + def _write_tool_call_item(item: object, name: str | None, payload: str | None) -> None: if item is None: return if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) - if arguments is not None: - OpenAIResponsesHandler._write_event_field(item, "arguments", arguments) + item_type: Final = stream_item_field(item, "type") + if payload is not None and isinstance(item_type, str) and item_type in _TOOL_CALL_PAYLOAD_FIELDS: + OpenAIResponsesHandler._write_event_field(item, _TOOL_CALL_PAYLOAD_FIELDS[item_type], payload) def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ @@ -1073,7 +1124,7 @@ class OpenAIResponsesHandler(BaseTranslation): def _completed_response_scan_key(response: object) -> StreamingScanKey: output_items: Final = stream_item_items(response, "output") message_items: Final = tuple( - item for item in output_items if stream_item_field(item, "type") != "function_call" + item for item in output_items if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES ) return StreamingScanKey( texts=tuple( @@ -1085,7 +1136,7 @@ class OpenAIResponsesHandler(BaseTranslation): tool_calls=tuple( stream_item_fingerprint(item) for item in output_items - if stream_item_field(item, "type") == "function_call" + if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES ), stream_ended=True, ) @@ -1196,34 +1247,10 @@ class OpenAIResponsesHandler(BaseTranslation): Override this method to customize text/image/tool extraction logic. """ - # Check if this is a tool call (OutputFunctionToolCall) - if isinstance(output_item, OutputFunctionToolCall) or ( - isinstance(output_item, BaseModel) - and hasattr(output_item, "type") - and getattr(output_item, "type") == "function_call" - ): + tool_call_item: Final = _tool_call_output_item_mapping(output_item) + if tool_call_item is not None: if tool_calls_to_check is not None: - tool_call_dict = ( - LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call( - tool_call_item=output_item, - index=output_idx, - ) - ) - tool_calls_to_check.append(cast(ChatCompletionToolCallChunk, tool_call_dict)) - return - elif isinstance(output_item, dict) and output_item.get("type") == "function_call": - # Handle dict representation of tool call - if tool_calls_to_check is not None: - # Convert dict to ResponseFunctionToolCall for processing - try: - tool_call_obj: Final = ResponseFunctionToolCall(**output_item) - tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call( - tool_call_item=tool_call_obj, - index=output_idx, - ) - tool_calls_to_check.append(cast(ChatCompletionToolCallChunk, tool_call_dict)) - except Exception: - pass + tool_calls_to_check.append(tool_call_dict_from_output_item(tool_call_item, output_idx)) return # Handle both GenericResponseOutputItem and dict diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 6ed4ec6618f..628ada05267 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -14,7 +14,12 @@ import pytest from fastapi import HTTPException -from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses import ( + ResponseCustomToolCall, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDoneEvent, + ResponseFunctionToolCall, +) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -27,7 +32,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.responses.main import GenericResponseOutputItem, OutputText +from litellm.types.responses.main import CustomToolCallOutputItem, GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs @@ -57,6 +62,37 @@ class MockGuardrail(CustomGuardrail): return inputs +class PersimmonMaskingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + tool_calls = [ + { + **tool_call, + "function": { + **tool_call["function"], + "arguments": tool_call["function"]["arguments"].replace("persimmon", "[MASKED]"), + }, + } + for tool_call in inputs.get("tool_calls", []) + ] + return {**inputs, "tool_calls": tool_calls} + + +CUSTOM_TOOL_CALL_ITEM = { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_exec_1", + "name": "exec", + "input": "echo persimmon", + "status": "completed", +} + + class TestOpenAIResponsesHandlerDiscovery: """Test that the handler is properly discovered by the guardrail system""" @@ -628,6 +664,77 @@ class TestOpenAIResponsesHandlerToolCallExtraction: == '{"location":"Boston, MA","unit":"celsius"}' ) + @pytest.mark.parametrize( + "output_item", + [ + dict(CUSTOM_TOOL_CALL_ITEM), + CustomToolCallOutputItem(**CUSTOM_TOOL_CALL_ITEM), + ResponseCustomToolCall(**{key: value for key, value in CUSTOM_TOOL_CALL_ITEM.items() if key != "status"}), + ], + ids=["dict", "litellm_typed", "openai_typed"], + ) + def test_extract_custom_tool_call_input_as_arguments(self, output_item): + handler = OpenAIResponsesHandler() + texts_to_check: List[str] = [] + tool_calls_to_check: List[Any] = [] + + handler._extract_output_text_and_images( + output_item=output_item, + output_idx=2, + texts_to_check=texts_to_check, + images_to_check=[], + task_mappings=[], + tool_calls_to_check=tool_calls_to_check, + ) + + assert texts_to_check == [] + assert tool_calls_to_check == [ + { + "id": "call_exec_1", + "type": "function", + "function": {"name": "exec", "arguments": "echo persimmon"}, + "index": 2, + } + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("typed", [False, True], ids=["dict", "typed"]) + async def test_process_output_response_writes_tool_call_rewrites_back(self, typed): + handler = OpenAIResponsesHandler() + function_call = { + "type": "function_call", + "id": "fc_1", + "call_id": "call_fn_1", + "name": "lookup_fruit", + "arguments": '{"fruit": "persimmon"}', + "status": "completed", + } + message = { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "running persimmon", "annotations": []}], + } + payload = { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [message, function_call, dict(CUSTOM_TOOL_CALL_ITEM)], + } + response = ResponsesAPIResponse.model_validate(payload) if typed else payload + + result = await handler.process_output_response(response, PersimmonMaskingGuardrail(guardrail_name="mask")) + + output = result.output if typed else result["output"] + function_item, custom_item = output[1], output[2] + assert (function_item.arguments if typed else function_item["arguments"]) == '{"fruit": "[MASKED]"}' + assert (custom_item.input if typed else custom_item["input"]) == "echo [MASKED]" + assert (custom_item.name if typed else custom_item["name"]) == "exec" + assert (output[0].content[0].text if typed else output[0]["content"][0]["text"]) == "running persimmon" + @pytest.mark.asyncio async def test_process_output_response_with_tool_calls(self): """Test processing output response containing function tool calls""" @@ -1315,6 +1422,109 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' assert completed_event.response.output[0].name == "lookup_fruit" + @staticmethod + def _ended_custom_tool_call_stream_events() -> List[dict]: + def item(input_text: str, status: str) -> dict: + return {**CUSTOM_TOOL_CALL_ITEM, "input": input_text, "status": status} + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.custom_tool_call_input.delta", "item_id": "ctc_1", "output_index": 0, "delta": "echo "}, + {"type": "response.custom_tool_call_input.delta", "item_id": "ctc_1", "output_index": 0, "delta": "persimmon"}, + {"type": "response.custom_tool_call_input.done", "item_id": "ctc_1", "output_index": 0, "input": "echo persimmon"}, + {"type": "response.output_item.done", "output_index": 0, "item": item("echo persimmon", "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "created_at": 1, + "model": "gpt-5.6", + "output": [item("echo persimmon", "completed")], + "status": "completed", + }, + }, + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_custom_tool_call_events(self): + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + + result = await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is events + assert events[0]["item"]["input"] == "" + assert events[1]["delta"] == "echo [MASKED]" + assert events[2]["delta"] == "" + assert events[3]["input"] == "echo [MASKED]" + assert events[4]["item"]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["name"] == "exec" + assert "arguments" not in events[5]["response"]["output"][0] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_typed_custom_tool_call_events(self): + from litellm.types.llms.openai import ( + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ) + + handler = OpenAIResponsesHandler() + typed_events: List[Any] = [ + model.model_validate({**event, "sequence_number": sequence_number}) + for sequence_number, (model, event) in enumerate( + zip( + ( + OutputItemAddedEvent, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDoneEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ), + self._ended_custom_tool_call_stream_events(), + ) + ) + ] + completed_event = typed_events[5] + assert isinstance(completed_event.response.output[0], CustomToolCallOutputItem) + + await handler.process_output_streaming_response( + responses_so_far=typed_events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert typed_events[1].delta == "echo [MASKED]" + assert typed_events[2].delta == "" + assert typed_events[3].input == "echo [MASKED]" + assert typed_events[4].item.input == "echo [MASKED]" + assert completed_event.response.output[0].input == "echo [MASKED]" + assert completed_event.response.output[0].name == "exec" + + @pytest.mark.asyncio + async def test_deliver_ended_stream_custom_tool_call_rewrite_without_matching_events_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + events[5]["response"]["output"] = [{**events[5]["response"]["output"][0], "call_id": "call_999"}] + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @staticmethod def _bridged_function_call_stream_events() -> List[dict]: reasoning = {"type": "reasoning", "id": "rs_1", "summary": []} @@ -2747,8 +2957,21 @@ class TestOpenAIResponsesHandlerStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] assert ended_key != open_key + def test_completed_event_with_a_custom_tool_call_changes_the_key(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + ended_key = handler.get_streaming_scan_key( + [self._delta(0, "hi"), self._completed(1, [message, dict(CUSTOM_TOOL_CALL_ITEM)])] + ) + rewritten_key = handler.get_streaming_scan_key( + [self._delta(0, "hi"), self._completed(1, [message, {**CUSTOM_TOOL_CALL_ITEM, "input": "echo kumquat"}])] + ) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "echo persimmon" in ended_key.tool_calls[0] + assert rewritten_key != ended_key + def test_completed_event_reads_every_output_text_part(self): - from litellm.types.responses.main import GenericResponseOutputItem, OutputText + from litellm.types.responses.main import CustomToolCallOutputItem, GenericResponseOutputItem, OutputText item = GenericResponseOutputItem( type="message", From 5bdf45726d0a27372c1cc34ee6baa85ef5439650 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:52:08 -0700 Subject: [PATCH 49/81] fix(bedrock): honor the json_mode kwarg on invoke Nova structured output and keep gpt-4o-mini off chat web search --- .../bedrock/chat/converse_transformation.py | 10 +++-- ...odel_prices_and_context_window_backup.json | 6 +-- model_prices_and_context_window.json | 6 +-- .../chat/test_converse_transformation.py | 38 +++++++++++++++++++ whitelisted_bedrock_models.txt | 1 + 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fa24f8be893..6173f08fc87 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1805,6 +1805,7 @@ class AmazonConverseConfig(BaseConfig): data=request_data, messages=messages, encoding=encoding, + json_mode=json_mode, ) def _transform_reasoning_content(self, reasoning_content_blocks: list[BedrockConverseReasoningContentBlock]) -> str: @@ -2237,6 +2238,7 @@ class AmazonConverseConfig(BaseConfig): data: dict | str, messages: list, encoding, + json_mode: bool | None = None, ) -> ModelResponse: ## LOGGING if logging_obj is not None: @@ -2247,7 +2249,9 @@ class AmazonConverseConfig(BaseConfig): additional_args={"complete_input_dict": data}, ) - json_mode: Final[bool | None] = optional_params.get("json_mode", None) + resolved_json_mode: Final[bool | None] = ( + json_mode if json_mode is not None else optional_params.get("json_mode", None) + ) ## RESPONSE OBJECT try: completion_response: Final = ConverseResponseBlock(**response.json()) @@ -2339,7 +2343,7 @@ class AmazonConverseConfig(BaseConfig): chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str filtered_tools: Final = self._filter_json_mode_tools( - json_mode=json_mode, + json_mode=resolved_json_mode, tools=tools, chat_completion_message=chat_completion_message, ) @@ -2363,7 +2367,7 @@ class AmazonConverseConfig(BaseConfig): # When json_mode filtered out all synthetic tool calls the response # is plain content, not a pending tool invocation. Fix finish_reason # so callers (e.g. OpenAI SDK) don't misinterpret it. - if json_mode and not filtered_tools and tools: + if resolved_json_mode and not filtered_tools and tools: initial_finish_reason = "stop" ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 885021692c9..a2e7b692649 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29407,8 +29407,7 @@ "search_context_size_high": 0.025, "search_context_size_low": 0.025, "search_context_size_medium": 0.025 - }, - "supports_web_search": true + } }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29436,8 +29435,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true + "supports_vision": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 885021692c9..a2e7b692649 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29407,8 +29407,7 @@ "search_context_size_high": 0.025, "search_context_size_low": 0.025, "search_context_size_medium": 0.025 - }, - "supports_web_search": true + } }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29436,8 +29435,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true + "supports_vision": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f0e361ceb88..16824d19f23 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6727,3 +6727,41 @@ def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( ) assert result == {"any": {}} + + +def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it(): + response_json = { + "metrics": {"latencyMs": 900}, + "output": { + "message": { + "content": [ + { + "toolUse": { + "input": {"city": "Paris", "population": 2100000}, + "name": "json_tool_call", + "toolUseId": "tooluse_invoke_nova_json", + } + } + ], + "role": "assistant", + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 40, "outputTokens": 20, "totalTokens": 60}, + } + raw_response = httpx.Response(200, json=response_json, request=httpx.Request("POST", "https://bedrock.test")) + logging_obj = MagicMock() + result = AmazonConverseConfig().transform_response( + model="bedrock/invoke/us.amazon.nova-micro-v1:0", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=logging_obj, + request_data={}, + messages=[], + optional_params={"tools": [{"type": "function", "function": {"name": "json_tool_call", "parameters": {}}}]}, + litellm_params={}, + encoding=None, + json_mode=True, + ) + assert result.choices[0].message.tool_calls is None + assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000} diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 578edddec8d..6124cb41044 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -6,6 +6,7 @@ ai21.jamba-instruct-v1:0 twelvelabs.pegasus-1-2-v1:0 us.twelvelabs.pegasus-1-2-v1:0 eu.twelvelabs.pegasus-1-2-v1:0 +global.twelvelabs.pegasus-1-2-v1:0 amazon.titan-text-express-v1 amazon.titan-text-lite-v1 amazon.titan-text-premier-v1:0 From b89c32407f81141e6c877b534862ad5b3e7956e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:01:36 -0700 Subject: [PATCH 50/81] test(cost): assert gpt-4o-mini keeps chat web search off while the flat search fee still matches its alias --- .../llm_cost_calc/test_tool_call_cost_tracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index f319413045f..bbb7b5f9c35 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -879,8 +879,8 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( alias_info = litellm.get_model_info("gpt-4o-mini") snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") - assert snapshot_info["supports_web_search"] is True - assert alias_info["supports_web_search"] is True + assert not snapshot_info["supports_web_search"] + assert not alias_info["supports_web_search"] snapshot_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( web_search_options=web_search_options, model_info=snapshot_info From 29af8b734913f5e43e4149bcdcc36a7c9261cc37 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:04:54 -0700 Subject: [PATCH 51/81] fix(streaming_handler): replay a cached completion with no choices as an empty stream A stream cache hit on an entry stored with choices == [] indexed choices[0] in the cached_response branch and failed with IndexError, so the streaming converters' empty chunk had no working consumer. The branch now treats a chunk without choices as empty and lets the wrapper close the stream with its usual finish_reason stop chunk --- .../litellm_core_utils/streaming_handler.py | 11 ++--- .../test_streaming_handler.py | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6ae17bac6ff..db23929e0c3 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1473,17 +1473,14 @@ class CustomStreamWrapper: self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider == "cached_response": cached_chunk: Final = cast(ModelResponseStream, chunk) - chunk_finish_reason: Final = cached_chunk.choices[0].finish_reason + cached_choice: Final = cached_chunk.choices[0] if cached_chunk.choices else None + chunk_finish_reason: Final = cached_choice.finish_reason if cached_choice is not None else None response_obj = { - "text": cached_chunk.choices[0].delta.content, + "text": cached_choice.delta.content if cached_choice is not None else None, "is_finished": chunk_finish_reason is not None, "finish_reason": chunk_finish_reason, "original_chunk": cached_chunk, - "tool_calls": ( - cached_chunk.choices[0].delta.tool_calls - if hasattr(cached_chunk.choices[0].delta, "tool_calls") - else None - ), + "tool_calls": (getattr(cached_choice.delta, "tool_calls", None) if cached_choice is not None else None), } completion_obj["content"] = response_obj["text"] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 0aa73833677..ced7d6d677d 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -6,7 +6,7 @@ import pytest import asyncio import traceback -from typing import Optional +from typing import Final, Optional import litellm from litellm import verbose_logger @@ -2633,6 +2633,47 @@ def test_dispatch_cached_response_extracts_delta( assert initialized_custom_stream_wrapper.response_id == "chatcmpl-cache-1" +def test_dispatch_cached_response_without_choices_is_an_empty_chunk( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """A cached completion with no choices replays as an empty, unfinished chunk + instead of raising IndexError on choices[0].""" + initialized_custom_stream_wrapper.custom_llm_provider = "cached_response" + chunk: Final = ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + result, model_response, completion_obj = _run_dispatch( + initialized_custom_stream_wrapper, chunk + ) + + assert isinstance(result, _ProviderChunkParsed) + assert completion_obj["content"] is None + assert initialized_custom_stream_wrapper.received_finish_reason is None + assert model_response.id == "chatcmpl-cache-empty" + + +@pytest.mark.asyncio +async def test_cached_response_without_choices_streams_a_single_stop_chunk( + logging_obj: Logging, +): + """A stream cache hit on a completion stored with choices == [] ends with one + finish_reason=stop chunk, the same shape the live empty stream produced.""" + + async def cached_chunks(): + yield ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + wrapper: Final = CustomStreamWrapper( + completion_stream=cached_chunks(), + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="cached_response", + ) + + chunks: Final = tuple([chunk async for chunk in wrapper]) + + assert tuple(choice.finish_reason for chunk in chunks for choice in chunk.choices) == ("stop",) + assert all(choice.delta.content in (None, "") for chunk in chunks for choice in chunk.choices) + + def test_dispatch_vertex_ai_legacy_text_and_finish_reason( initialized_custom_stream_wrapper: CustomStreamWrapper, ): From 354365eecaf6c70aba2a649fa324bf0369438f85 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:15:01 -0700 Subject: [PATCH 52/81] test(proxy): give the pipeline-managed native hook audit test its chat completions route --- tests/test_litellm/proxy/test_proxy_logging_hook_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 42888ca7d54..28ff4571b44 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -688,7 +688,7 @@ async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monk "messages": [{"role": "user", "content": "hi"}], "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, }, - captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"), + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"), captured_logging_obj=_streaming_logging_obj(), assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), cache_hit=False, From 0a053d2c8146e4a4739f92d48f1e2bdec75202e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:21:40 -0700 Subject: [PATCH 53/81] test(guardrails): cover streamed tool-call name rewrites on chat and Messages Skipping the name write-back in either handler left every test green; a guardrail that renames a tool call now has a regression test on both the chat chunk path and the Anthropic SSE path --- .../test_anthropic_guardrail_handler.py | 23 +++++++++++++++++++ .../test_openai_guardrail_handler.py | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 64c243343d5..e091355b69d 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -367,6 +367,29 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert '"stop_reason": "tool_use"' in raw assert "persim" not in raw + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_use_name_back_into_sse_chunks(self): + class RenameTool(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call.function.name = "lookup_fruit_reviewed" + return inputs + + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=RenameTool(guardrail_name="test"), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert '"name": "lookup_fruit_reviewed"' in raw and '"id": "toolu_1"' in raw + assert '"name": "lookup_fruit"' not in raw + assert json.loads("".join(self._partial_jsons(chunks))) == {"fruit": "persimmon"} + @pytest.mark.asyncio async def test_ended_stream_tool_use_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c7011a6f00e..5a29a96829f 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1171,6 +1171,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[3].choices[0].delta.tool_calls is None assert chunks[3].choices[0].finish_reason == "tool_calls" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_call_name_back_into_chunks(self): + class RenameTool(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call["function"]["name"] = "lookup_fruit_reviewed" + return inputs + + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=RenameTool(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + fragments = [chunk.choices[0].delta.tool_calls[0] for chunk in chunks[:3]] + assert [fragment.function.name for fragment in fragments] == ["lookup_fruit_reviewed", None, None] + assert json.loads("".join(fragment.function.arguments for fragment in fragments)) == {"fruit": "persimmon"} + assert fragments[0].id == "call_1" + @pytest.mark.asyncio async def test_ended_stream_tool_call_rewrite_leaves_chunks_untouched_by_default(self): handler = OpenAIChatCompletionsHandler() From cce1d2087b9ae997ec406d6d38857642f1461844 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:22:22 -0700 Subject: [PATCH 54/81] fix(responses): echo a named tool_choice in the Responses API shape on the chat-completions bridge A streamed /v1/responses request with tool_choice {"type": "function", "name": ...} that reaches a chat-completions-only deployment failed with HTTP 500 before the first byte: the synthetic response.created and response.in_progress events copied the chat-shaped tool_choice into ResponsesAPIResponse, whose ToolChoice type expects the flat Responses API shape. The non-streamed path echoed "auto" regardless of the request. Both paths now normalize the request's tool_choice through the existing chat transform and map it back to the Responses API vocabulary, validated by a TypeAdapter(ToolChoice), so a named function is echoed as {"type": "function", "name": ...} and a missing tool_choice is echoed as "auto". Fixes #33689 --- .../streaming_iterator.py | 11 ++-- .../transformation.py | 18 ++++++- .../test_litellm_completion_responses.py | 52 ++++++++++++++++++ .../test_streaming_iterator_transformation.py | 53 +++++++++++++++++++ 4 files changed, 126 insertions(+), 8 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index d7f8cd8f8bd..660dd8f0c92 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -437,14 +437,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): response_created_event_data["temperature"] = self.responses_api_request["temperature"] if "text" in self.responses_api_request: response_created_event_data["text"] = self.responses_api_request["text"] - if "tool_choice" in self.responses_api_request: - # Transform tool_choice from dict format (e.g., {"type": "auto"}) to string format - response_created_event_data["tool_choice"] = ( - LiteLLMCompletionResponsesConfig._transform_tool_choice(self.responses_api_request["tool_choice"]) - or "auto" + response_created_event_data["tool_choice"] = ( + LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + self.responses_api_request.get("tool_choice") ) - else: - response_created_event_data["tool_choice"] = "auto" + ) if "tools" in self.responses_api_request: response_created_event_data["tools"] = self.responses_api_request["tools"] else: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b2d1a69e0d8..878f493b58e 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -27,6 +27,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( ) from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -68,6 +69,7 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStatus, + ToolChoice, ValidChatCompletionMessageContentTypes, ValidChatCompletionMessageContentTypesLiteral, ) @@ -126,6 +128,7 @@ _STR_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]]) _TEXT_ADAPTER: Final = TypeAdapter(str) +_RESPONSES_API_TOOL_CHOICE_ADAPTER: Final = TypeAdapter(ToolChoice) @runtime_checkable @@ -267,6 +270,17 @@ class LiteLLMCompletionResponsesConfig: # Return as-is for unknown formats return tool_choice + @staticmethod + def _transform_tool_choice_for_responses_api_response(tool_choice: object) -> ToolChoice: + normalized: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice) + match normalized: + case None: + return "auto" + case {"type": "function", "function": {"name": str(function_name)}}: + return ToolChoiceFunctionParam(type="function", name=function_name) + case _: + return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) + @staticmethod def _should_drop_derived_web_search_options(model: str, custom_llm_provider: str | None) -> bool: """ @@ -2263,7 +2277,9 @@ class LiteLLMCompletionResponsesConfig: ), parallel_tool_calls=getattr(chat_completion_response, "parallel_tool_calls", False), temperature=getattr(chat_completion_response, "temperature", 0), - tool_choice=getattr(chat_completion_response, "tool_choice", "auto"), + tool_choice=LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + responses_api_request.get("tool_choice") + ), tools=getattr(chat_completion_response, "tools", []), top_p=getattr(chat_completion_response, "top_p", None), max_output_tokens=getattr(chat_completion_response, "max_output_tokens", None), diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2068f10ea2d..2aab660b5b7 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1421,6 +1421,58 @@ class TestToolChoiceTransformation: ) assert result == "required" + @pytest.mark.parametrize( + "request_tool_choice,expected", + [ + ({"type": "function", "name": "run_command"}, {"type": "function", "name": "run_command"}), + ({"type": "function", "function": {"name": "run_command"}}, {"type": "function", "name": "run_command"}), + ({"type": "custom", "name": "ApplyPatch"}, {"type": "function", "name": "ApplyPatch"}), + ({"type": "tool"}, "required"), + ({"type": "auto"}, "auto"), + ("required", "required"), + ("none", "none"), + (None, "auto"), + ], + ) + def test_transform_tool_choice_for_responses_api_response(self, request_tool_choice, expected): + result = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + request_tool_choice + ) + assert result == expected + + def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self): + chat_completion_response = ModelResponse( + id="chatcmpl-named-tool-choice", + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion", + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_pwd", + type="function", + function=Function(name="run_command", arguments='{"command":"pwd"}'), + ) + ], + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Run the command pwd.", + responses_api_request={"tool_choice": {"type": "function", "name": "run_command"}}, + chat_completion_response=chat_completion_response, + ) + + assert responses_api_response.tool_choice == {"type": "function", "name": "run_command"} + class TestContentTypeTransformation: """Test content type transformation from Responses API to Chat Completion format""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 719d51c11e3..d4b565f82a1 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -628,3 +628,56 @@ def test_streamed_anthropic_tool_call_events_correlate_on_normalized_item_id(): assert item_dones[0].item.call_id == "toolu_01AbCdEf" for evt in deltas + dones: assert evt.item_id == added[0].item.id + + +def _tool_call_chunk(finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_pwd", + "type": "function", + "function": {"name": "run_command", "arguments": '{"command":"pwd"}'}, + "index": 0, + } + ], + ), + finish_reason=finish_reason, + ) + ], + ) + + +def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): + iterator = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), + request_input="Run the command pwd.", + responses_api_request={ + "tools": [{"type": "function", "name": "run_command", "parameters": {"type": "object"}}], + "tool_choice": {"type": "function", "name": "run_command"}, + }, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + + events = list(iterator) + + response_events = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] + assert [event.type for event in response_events] == [ + "response.created", + "response.in_progress", + "response.completed", + ] + assert response_events[0].response.tool_choice == {"type": "function", "name": "run_command"} + assert response_events[1].response.tool_choice == {"type": "function", "name": "run_command"} + assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) From 5e7cb898065158626fec3a4afb90cb23a411095c Mon Sep 17 00:00:00 2001 From: Praveena Date: Tue, 4 Aug 2026 16:33:28 +0530 Subject: [PATCH 55/81] fix(azure): respect DEFAULT_MAX_RETRIES in initialize_azure_sdk_client When litellm_params does not include max_retries (the common case for deployments configured without explicit retry settings), initialize_azure_sdk_client() previously passed None through the 'if max_retries is not None' guard, resulting in AsyncAzureOpenAI being created without a max_retries argument. The OpenAI SDK then uses its own hardcoded default of 2, ignoring the DEFAULT_MAX_RETRIES env var. Fix: fall back to litellm.constants.DEFAULT_MAX_RETRIES when litellm_params has no max_retries. This ensures the SDK client respects the configured retry count. Steps to reproduce: 1. Set env var DEFAULT_MAX_RETRIES=0 2. Configure a deployment without explicit max_retries in litellm_params 3. Make a request that triggers a timeout 4. Observe: SDK retries (x-stainless-retry-count=1) despite env var=0 Related: https://github.com/BerriAI/litellm/issues/5124 --- litellm/llms/azure/common_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 6cb7d09cec4..f20cc37572b 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -582,7 +582,11 @@ class BaseAzureLLM(BaseOpenAILLM): if scope is None: scope = "https://cognitiveservices.azure.com/.default" - max_retries: Final = litellm_params.get("max_retries") + max_retries = litellm_params.get("max_retries") + if max_retries is None: + from litellm.constants import DEFAULT_MAX_RETRIES + + max_retries = DEFAULT_MAX_RETRIES timeout: Final = litellm_params.get("timeout") if not api_key and azure_ad_token_provider is None and tenant_id and client_id and client_secret: verbose_logger.debug("Using Azure AD Token Provider from Entra ID for Azure Auth") From 830ee2d39d84eba4d2c8a27851e363473af038d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:29:06 -0700 Subject: [PATCH 56/81] fix(azure): default max_retries to DEFAULT_MAX_RETRIES with regression tests initialize_azure_sdk_client now falls back to litellm.constants.DEFAULT_MAX_RETRIES when litellm_params carries no max_retries, so off-router Azure clients (files, batches, fine-tuning, assistants, audio) honor the env var like OpenAI clients do. Router paths already default max_retries to 0 and are unchanged. Regression tests cover the default, explicit 0/5/None values, and the env var reaching the SDK client in a fresh interpreter. --- litellm/llms/azure/common_utils.py | 11 ++-- .../llms/azure/test_azure_common_utils.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f20cc37572b..e6b3eb1f2bb 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -14,6 +14,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import DualCache +from litellm.constants import DEFAULT_MAX_RETRIES from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.common_utils import BaseOpenAILLM from litellm.secret_managers.get_azure_ad_token_provider import ( @@ -582,11 +583,8 @@ class BaseAzureLLM(BaseOpenAILLM): if scope is None: scope = "https://cognitiveservices.azure.com/.default" - max_retries = litellm_params.get("max_retries") - if max_retries is None: - from litellm.constants import DEFAULT_MAX_RETRIES - - max_retries = DEFAULT_MAX_RETRIES + configured_max_retries: Final = litellm_params.get("max_retries") + max_retries: Final = DEFAULT_MAX_RETRIES if configured_max_retries is None else configured_max_retries timeout: Final = litellm_params.get("timeout") if not api_key and azure_ad_token_provider is None and tenant_id and client_id and client_secret: verbose_logger.debug("Using Azure AD Token Provider from Entra ID for Azure Auth") @@ -646,8 +644,7 @@ class BaseAzureLLM(BaseOpenAILLM): else: azure_client_params["http_client"] = self._get_sync_http_client() - if max_retries is not None: - azure_client_params["max_retries"] = max_retries + azure_client_params["max_retries"] = max_retries if timeout is not None: azure_client_params["timeout"] = timeout diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index f000abb4c9a..c959c201ccb 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -385,6 +385,58 @@ def test_select_azure_base_url_called(setup_mocks): setup_mocks["select_url"].assert_called_once() +def test_initialize_defaults_max_retries_to_litellm_default(setup_mocks): + result = BaseAzureLLM().initialize_azure_sdk_client( + litellm_params={}, + api_key="test-api-key", + api_base="https://test.openai.azure.com", + model_name="gpt-4", + api_version="2023-06-01", + is_async=False, + ) + + assert result["max_retries"] == litellm.constants.DEFAULT_MAX_RETRIES + + +@pytest.mark.parametrize( + "configured, expected", + [(0, 0), (5, 5), (None, litellm.constants.DEFAULT_MAX_RETRIES)], +) +def test_initialize_honors_explicit_max_retries(setup_mocks, configured, expected): + result = BaseAzureLLM().initialize_azure_sdk_client( + litellm_params={"max_retries": configured}, + api_key="test-api-key", + api_base="https://test.openai.azure.com", + model_name="gpt-4", + api_version="2023-06-01", + is_async=False, + ) + + assert result["max_retries"] == expected + + +def test_default_max_retries_env_var_reaches_azure_sdk_client(): + import subprocess + import sys + + code = ( + "from litellm.llms.azure.common_utils import BaseAzureLLM\n" + "client = BaseAzureLLM().get_azure_openai_client(" + "api_key='test-api-key', api_base='https://test.openai.azure.com', api_version='2024-02-01'," + " client=None, _is_async=True, litellm_params={}, model='gpt-4')\n" + "print(client.max_retries)" + ) + completed = subprocess.run( + [sys.executable, "-c", code], + env={**os.environ, "DEFAULT_MAX_RETRIES": "0"}, + capture_output=True, + text=True, + check=True, + ) + + assert completed.stdout.strip() == "0" + + @pytest.mark.parametrize( "call_type", [ From 2488f84b0293956b3736fcd9cf639491ab68e65e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:53:35 -0700 Subject: [PATCH 57/81] fix(proxy): keep a body litellm_session_id in SpendLogs under missing_session_id omit (#40379) * fix(proxy): keep a body litellm_session_id in SpendLogs under missing_session_id omit Under general_settings.missing_session_id: omit, apply_missing_session_id_policy now mirrors a client-supplied top-level litellm_session_id into metadata.session_id when the client did not set one there, so SpendLogs.session_id and Langfuse agree with the session callbacks already report through StandardLoggingPayload.session_id Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep client metadata.session_id ahead of body litellm_session_id on litellm_metadata routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop docstrings from the missing_session_id omit regression tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 10 ++ .../proxy/test_litellm_pre_call_utils.py | 101 ++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3250ae5cca9..2925226f235 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -773,6 +773,16 @@ def apply_missing_session_id_policy( return if policy == "omit": metadata[SESSION_ID_OMITTED_METADATA_KEY] = True + requester_metadata: Final = data.get("metadata") + requester_session_id: Final = ( + requester_metadata.get("session_id") if isinstance(requester_metadata, dict) else None + ) + if ( + (body_session_id := data.get("litellm_session_id")) + and not metadata.get("session_id") + and not requester_session_id + ): + metadata["session_id"] = body_session_id return if data.get("litellm_session_id") or metadata.get("session_id"): return diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 892fa484ab4..7564fa91c43 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -7678,6 +7678,107 @@ async def test_missing_session_id_omit_keeps_client_supplied_session_id(): assert _spend_log_session_id(updated) == "client-session-1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_body", + [ + {"model": "gpt-4o", "messages": [], "litellm_session_id": "cust-sess-1"}, + {"model": "gpt-4o", "messages": [], "litellm_session_id": "cust-sess-1", "metadata": {"trace_id": "trace-1"}}, + ], +) +async def test_missing_session_id_omit_keeps_body_litellm_session_id( + monkeypatch: pytest.MonkeyPatch, client_body: dict[str, object] +): + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + + updated = await add_litellm_data_to_request( + data=client_body, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + callback_session_id = StandardLoggingPayloadSetup.get_standard_logging_payload_session_id( + logging_obj=SimpleNamespace(litellm_session_id=""), + litellm_params=get_litellm_params(litellm_session_id="cust-sess-1", metadata=updated["metadata"]), + ) + assert callback_session_id == "cust-sess-1" + assert updated["metadata"]["session_id"] == "cust-sess-1" + assert _spend_log_session_id(updated) == "cust-sess-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_omit_body_litellm_session_id_does_not_override_metadata_session_id(): + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "messages": [], + "litellm_session_id": "cust-sess-1", + "metadata": {"session_id": "meta-sess-1"}, + }, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["metadata"]["session_id"] == "meta-sess-1" + assert _spend_log_session_id(updated) == "meta-sess-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +async def test_missing_session_id_omit_keeps_metadata_session_id_on_litellm_metadata_routes(path: str): + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "input": "hi", + "litellm_session_id": "cust-sess-1", + "metadata": {"session_id": "meta-sess-1"}, + }, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["litellm_metadata"]["session_id"] == "meta-sess-1" + assert _spend_log_session_id(updated, "litellm_metadata") == "meta-sess-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +async def test_missing_session_id_omit_keeps_body_litellm_session_id_on_litellm_metadata_routes(path: str): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "input": "hi", "litellm_session_id": "cust-sess-1"}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["litellm_metadata"]["session_id"] == "cust-sess-1" + assert _spend_log_session_id(updated, "litellm_metadata") == "cust-sess-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_omit_ignores_empty_body_litellm_session_id(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], "litellm_session_id": ""}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert "session_id" not in updated["metadata"] + assert _spend_log_session_id(updated) is None + + @pytest.mark.asyncio async def test_missing_session_id_generate_reuses_traceparent_trace_id(): """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" From cbeac276561c82e71f56de6d76e4d132170a2cc6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:12:13 -0700 Subject: [PATCH 58/81] fix(convert_dict_to_response): name the non-list choices type in the converter error When a provider returns choices as null, an object, a string or a number, the converter said the response had no 'choices' even though the key was present in the raw keys it listed. A shared message now keeps the old wording for a missing key and names the offending type otherwise. The cached-stream regression test also pins the chunk count so a leaked extra chunk fails it. --- .../convert_dict_to_response.py | 25 +++++++++++-------- .../test_convert_dict_to_response.py | 11 ++++---- .../test_streaming_handler.py | 1 + 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 29b75812bfb..87524d86c61 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,7 +3,7 @@ import json import re import time import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final, Literal, cast import litellm @@ -151,6 +151,16 @@ def _clear_later_replay_slice_metadata(choice: StreamingChoices) -> None: del choice.enhancements +def _invalid_choices_message(response_object: Mapping[str, object]) -> str: + raw_keys: Final = list(response_object.keys()) + if "choices" not in response_object: + return f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {raw_keys}" + return ( + f"LiteLLM: provider returned 'choices' that is not a list ({type(response_object['choices']).__name__}). " + f"Raw keys: {raw_keys}" + ) + + async def convert_to_streaming_response_async( response_object: dict | None = None, ): @@ -184,9 +194,7 @@ async def convert_to_streaming_response_async( raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -292,9 +300,7 @@ def convert_to_streaming_response( raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -628,10 +634,7 @@ def convert_to_model_response_object( raise APIError( status_code=500, - message=( - "LiteLLM: provider returned a response with no 'choices'. " - f"Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index c3e99f01cec..8e46ae21de6 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -167,9 +167,9 @@ def test_convert_missing_choices_raises_api_error() -> None: assert "no 'choices'" in str(exc_info.value) -@pytest.mark.parametrize("choices", [{}, "", None, 0]) +@pytest.mark.parametrize(("choices", "type_name"), [({}, "dict"), ("", "str"), (None, "NoneType"), (0, "int")]) @pytest.mark.asyncio -async def test_convert_non_list_choices_raises_api_error(choices: object) -> None: +async def test_convert_non_list_choices_raises_api_error(choices: object, type_name: str) -> None: from litellm.exceptions import APIError from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_streaming_response, @@ -183,14 +183,15 @@ async def test_convert_non_list_choices_raises_api_error(choices: object) -> Non "object": "chat.completion", "choices": choices, } - with pytest.raises(APIError, match="no 'choices'"): + expected: Final = f"'choices' that is not a list \\({type_name}\\)" + with pytest.raises(APIError, match=expected): convert_to_model_response_object( response_object=resp, model_response_object=ModelResponse(), response_type="completion", ) - with pytest.raises(APIError, match="no 'choices'"): + with pytest.raises(APIError, match=expected): list(convert_to_streaming_response(response_object=resp)) - with pytest.raises(APIError, match="no 'choices'"): + with pytest.raises(APIError, match=expected): async for _ in convert_to_streaming_response_async(response_object=resp): pass diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index ced7d6d677d..37e2031fdf4 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2670,6 +2670,7 @@ async def test_cached_response_without_choices_streams_a_single_stop_chunk( chunks: Final = tuple([chunk async for chunk in wrapper]) + assert len(chunks) == 1 assert tuple(choice.finish_reason for chunk in chunks for choice in chunk.choices) == ("stop",) assert all(choice.delta.content in (None, "") for chunk in chunks for choice in chunk.choices) From 264fc82dc1bb66e840b625830c9d8be03d655a0b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:14:10 -0700 Subject: [PATCH 59/81] fix(rate_limiter): attach v3 priority rate limit headers on /v1/messages (#37228) * fix(rate_limiter): attach v3 priority rate limit headers on /v1/messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(playground): honor the Stream responses toggle for /v1/messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rate_limiter): drop explanatory docstrings from v3 dict response tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 18 ++--- .../hooks/parallel_request_limiter_v3.py | 33 +++------- .../add_retry_fallback_headers.py | 6 ++ .../hooks/test_dynamic_rate_limiter_v3.py | 57 ++++++++++++++++ .../hooks/test_parallel_request_limiter_v3.py | 65 ++++++++++++++++++ .../components/chat_ui/ChatUI.test.tsx | 53 +++++++++++++++ .../playground/components/chat_ui/ChatUI.tsx | 6 +- .../llm_calls/anthropic_messages.test.tsx | 66 ++++++++++++++++++- .../llm_calls/anthropic_messages.tsx | 33 +++++++--- 9 files changed, 291 insertions(+), 46 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index de8834449de..a074f02f4e8 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -32,6 +32,10 @@ from litellm.proxy.hooks.rate_limiter_utils import ( resolve_llm_provider_for_rate_limit, ) from litellm.proxy.utils import InternalUsageCache +from litellm.router_utils.add_retry_fallback_headers import ( + ensure_response_additional_headers, + response_has_hidden_params, +) from litellm.types.router import ModelGroupInfo from litellm.types.utils import CallTypesLiteral @@ -659,22 +663,12 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): data=data, user_api_key_dict=user_api_key_dict, response=response ) - # Add additional priority-specific headers - if isinstance(response, ModelResponse): + if response_has_hidden_params(response): priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) - - # Get existing additional headers - additional_headers: Final = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {} - - # Add priority information + additional_headers: Final = ensure_response_additional_headers(response) additional_headers["x-litellm-priority"] = priority or "default" additional_headers["x-litellm-rate-limiter-version"] = "v3" - # Update response - if not hasattr(response, "_hidden_params"): - response._hidden_params = {} - response._hidden_params["additional_headers"] = additional_headers - return response except Exception as e: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 31437af7770..a72ae3bb1ea 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -52,6 +52,10 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.router_utils.add_retry_fallback_headers import ( + ensure_response_additional_headers, + response_has_hidden_params, +) from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( @@ -4677,34 +4681,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): Post-call hook to update rate limit headers in the response. """ try: - from pydantic import BaseModel - stash: Final = get_request_stash() litellm_proxy_rate_limit_response: Final = stash.rate_limit_response if stash is not None else None - if litellm_proxy_rate_limit_response is not None: - # Update response headers - if hasattr(response, "_hidden_params"): - _hidden_params = getattr(response, "_hidden_params") - else: - _hidden_params = None - - if _hidden_params is not None and ( - isinstance(_hidden_params, BaseModel) or isinstance(_hidden_params, dict) - ): - if isinstance(_hidden_params, BaseModel): - _hidden_params = _hidden_params.model_dump() - - _additional_headers: Final = self._merge_ratelimit_statuses_into_additional_headers( - additional_headers=_hidden_params.get("additional_headers", {}) or {}, + if litellm_proxy_rate_limit_response is not None and response_has_hidden_params(response): + additional_headers: Final = ensure_response_additional_headers(response) + additional_headers.update( + self._merge_ratelimit_statuses_into_additional_headers( + additional_headers={}, statuses=litellm_proxy_rate_limit_response["statuses"], ) - - setattr( - response, - "_hidden_params", - {**_hidden_params, "additional_headers": _additional_headers}, - ) + ) except Exception as e: verbose_proxy_logger.exception("Error in rate limit post-call hook: %s", e) diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index 3ec92ad226a..3251ea457cf 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -50,6 +50,12 @@ def prepare_response_for_header_attachment(response: object) -> object | None: return response +def response_has_hidden_params(response: object) -> bool: + if isinstance(response, dict): + return "_hidden_params" in response + return hasattr(response, "_hidden_params") + + def ensure_response_additional_headers(response: object) -> dict[str, object]: hidden_params: Final = get_hidden_params_dict(response, create=isinstance(response, dict)) _write_hidden_params(response, hidden_params) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0ff8b67b1a7..0cd6b4ede9c 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1861,3 +1861,60 @@ async def test_tpm_only_model_enforces_priority_and_model_capacity(monkeypatch): ) assert capacity_blocked.value.status_code == 429 assert "Model capacity reached" in capacity_blocked.value.detail["error"] + + +@pytest.mark.asyncio +async def test_post_call_success_hook_attaches_priority_headers_to_dict_response(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + RateLimitResponse, + RateLimitStatus, + get_or_create_request_stash, + ) + + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=75, + limit_remaining=74, + rate_limit_type="requests", + descriptor_key="priority_model", + ) + ], + ) + response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "_hidden_params": {"additional_headers": {"x-litellm-attempted-retries": 0}}, + } + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + assert additional_headers["x-litellm-attempted-retries"] == 0 + assert additional_headers["x-ratelimit-priority_model-limit-requests"] == 75 + assert additional_headers["x-ratelimit-priority_model-remaining-requests"] == 74 + assert additional_headers["x-litellm-priority"] == "premium" + assert additional_headers["x-litellm-rate-limiter-version"] == "v3" + + +@pytest.mark.asyncio +async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}), + response=response, + ) + + assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 4003286d887..6d382370f5f 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6171,3 +6171,68 @@ async def test_success_hook_leaves_stash_untouched_for_non_batch_responses(): data={}, user_api_key_dict=user, response=ModelResponse(usage=Usage(total_tokens=5)) ) assert get_request_stash().batch_enqueued_reservation == reservation + + +@pytest.mark.asyncio +async def test_post_call_success_hook_attaches_ratelimit_headers_to_dict_response(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitResponse, RateLimitStatus + + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=100, + limit_remaining=99, + rate_limit_type="requests", + descriptor_key="model_saturation_check", + ) + ], + ) + response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "_hidden_params": {"additional_headers": {"x-litellm-attempted-retries": 0}}, + } + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-dict-response")), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + assert additional_headers["x-litellm-attempted-retries"] == 0 + assert additional_headers["x-ratelimit-model_saturation_check-limit-requests"] == 100 + assert additional_headers["x-ratelimit-model_saturation_check-remaining-requests"] == 99 + + +@pytest.mark.asyncio +async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitResponse, RateLimitStatus + + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=100, + limit_remaining=99, + rate_limit_type="requests", + descriptor_key="model_saturation_check", + ) + ], + ) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-raw-dict")), + response=response, + ) + + assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx index f07b66efdf8..bf6b092a2b0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import ChatUI from "./ChatUI"; import * as fetchModelsModule from "@/components/llm_calls/fetch_models"; import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; +import { makeAnthropicMessagesRequest } from "../../llm_calls/anthropic_messages"; vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), @@ -14,6 +15,10 @@ vi.mock("@/components/llm_calls/chat_completion", () => ({ makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../../llm_calls/anthropic_messages", () => ({ + makeAnthropicMessagesRequest: vi.fn().mockResolvedValue(undefined), +})); + vi.mock("@/components/networking", () => ({ tagListCall: vi.fn().mockResolvedValue({}), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), @@ -32,6 +37,8 @@ beforeEach(() => { const CHAT_REQUEST_ARG_COUNT = 26; const STREAMING_ENABLED_ARG_INDEX = 25; +const MESSAGES_REQUEST_ARG_COUNT = 19; +const MESSAGES_STREAMING_ENABLED_ARG_INDEX = 18; async function openComboboxByPlaceholder(placeholder: string) { const user = userEvent.setup(); @@ -378,6 +385,52 @@ describe("ChatUI", () => { expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false); }); + it("should send the /v1/messages request non-streaming after Stream responses is unchecked", async () => { + const user = userEvent.setup(); + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + await selectComboboxOption("Select an endpoint", "/v1/messages"); + await selectComboboxOption("Select a Model", "Model 1"); + + await user.click(await screen.findByTestId("model-settings-button")); + + const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + await user.click(streamingCheckbox); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeAnthropicMessagesRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeAnthropicMessagesRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(MESSAGES_REQUEST_ARG_COUNT); + expect(requestArgs[MESSAGES_STREAMING_ENABLED_ARG_INDEX]).toBe(false); + }); + it("should force streaming in simplified mode even when the playground setting is off", async () => { sessionStorage.setItem("streamingEnabled", "false"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index 35378d3d4e7..eca9e2323ae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -1025,6 +1025,7 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, mcpToolsets, + streamingEnabled, ); } else if (endpointType === EndpointType.EMBEDDINGS) { await makeOpenAIEmbeddingsRequest( @@ -1174,7 +1175,10 @@ const ChatUI: React.FC = ({ return !model.mode || model.mode === "chat"; }; - const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES; + const supportsStreamingToggle = + endpointType === EndpointType.CHAT || + endpointType === EndpointType.RESPONSES || + endpointType === EndpointType.ANTHROPIC_MESSAGES; const modelsForEndpoint = useMemo( () => filterModelsForEndpoint(modelInfo, endpointType as EndpointType), [modelInfo, endpointType], diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx index 96ace129f87..9f030d8b2d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx @@ -7,13 +7,27 @@ vi.mock("@/components/networking", () => ({ })); const mockMessagesStream = vi.fn(); +const mockMessagesCreate = vi.fn(); vi.mock("@anthropic-ai/sdk", () => ({ default: vi.fn(function () { - return { messages: { stream: mockMessagesStream } }; + return { messages: { stream: mockMessagesStream, create: mockMessagesCreate } }; }), })); +const NON_STREAMING_ARGS = [ + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled +] as const; + describe("anthropic_messages prompt cache usage", () => { const captureUsage = async (usage: Record): Promise => { async function* mockStream() { @@ -59,3 +73,53 @@ describe("anthropic_messages prompt cache usage", () => { expect(usageData.promptTokens).toBe(5000); }); }); + +describe("anthropic_messages non-streaming", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("sends stream:false through messages.create and renders the full reply at once", async () => { + mockMessagesCreate.mockResolvedValue({ + content: [ + { type: "thinking", thinking: "considering" }, + { type: "text", text: "OK" }, + ], + usage: { input_tokens: 12, output_tokens: 3, cache_read_input_tokens: 7 }, + }); + const updateTextUI = vi.fn(); + const onReasoningContent = vi.fn(); + const onUsageData = vi.fn(); + + await makeAnthropicMessagesRequest( + [{ role: "user", content: "Hello" }], + updateTextUI, + "claude-haiku-4-5", + "test-token", + undefined, + undefined, + onReasoningContent, + undefined, + onUsageData, + ...NON_STREAMING_ARGS, + ); + + expect(mockMessagesStream).not.toHaveBeenCalled(); + expect(mockMessagesCreate).toHaveBeenCalledTimes(1); + expect(mockMessagesCreate.mock.calls[0][0]).toMatchObject({ model: "claude-haiku-4-5", stream: false }); + expect(updateTextUI).toHaveBeenCalledWith("assistant", "OK", "claude-haiku-4-5"); + expect(onReasoningContent).toHaveBeenCalledWith("considering"); + const expectedUsage: TokenUsage = { completionTokens: 3, promptTokens: 12, totalTokens: 15, cacheReadTokens: 7 }; + expect(onUsageData).toHaveBeenCalledWith(expectedUsage); + }); + + it("keeps streaming as the default when the flag is omitted", async () => { + async function* emptyStream() {} + mockMessagesStream.mockReturnValue(emptyStream()); + + await makeAnthropicMessagesRequest([{ role: "user", content: "Hello" }], vi.fn(), "claude-haiku-4-5", "test-token"); + + expect(mockMessagesCreate).not.toHaveBeenCalled(); + expect(mockMessagesStream.mock.calls[0][0]).toMatchObject({ stream: true }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx index 14afa013768..9dd2f675c44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx @@ -7,6 +7,13 @@ import { getProxyBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +const toTokenUsage = (usage: Anthropic.Usage): TokenUsage => ({ + completionTokens: usage.output_tokens, + promptTokens: usage.input_tokens, + totalTokens: usage.input_tokens + usage.output_tokens, + ...extractPromptCacheTokens(usage), +}); + export async function makeAnthropicMessagesRequest( messages: MessageType[], updateTextUI: (role: string, delta: string, model?: string) => void, @@ -26,6 +33,7 @@ export async function makeAnthropicMessagesRequest( mcpServers?: MCPServer[], mcpServerToolRestrictions?: Record, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -58,7 +66,7 @@ export async function makeAnthropicMessagesRequest( const requestBody: any = { model: selectedModel, messages: messages.map((m) => ({ role: m.role, content: m.content })), - stream: true, + stream: streamingEnabled, max_tokens: 1024, // @ts-ignore - litellm specific parameter litellm_trace_id: traceId, @@ -74,6 +82,20 @@ export async function makeAnthropicMessagesRequest( if (vector_store_ids) requestBody.vector_store_ids = vector_store_ids; if (guardrails) requestBody.guardrails = guardrails; if (policies) requestBody.policies = policies; + + if (!streamingEnabled) { + const message: Anthropic.Message = await client.messages.create({ ...requestBody, stream: false }, { signal }); + for (const block of message.content) { + if (block.type === "text") { + updateTextUI("assistant", block.text, selectedModel); + } else if (block.type === "thinking" && onReasoningContent) { + onReasoningContent(block.thinking); + } + } + onUsageData?.(toTokenUsage(message.usage)); + return; + } + // Use the streaming helper method for cleaner async iteration // @ts-ignore - The SDK types might not include all litellm-specific parameters const stream = client.messages.stream(requestBody, { signal }); @@ -105,14 +127,7 @@ export async function makeAnthropicMessagesRequest( // Process usage data from message_delta events if (messageStreamEvent.type === "message_delta" && (messageStreamEvent as any).usage && onUsageData) { - const usage = (messageStreamEvent as any).usage; - const usageData: TokenUsage = { - completionTokens: usage.output_tokens, - promptTokens: usage.input_tokens, - totalTokens: usage.input_tokens + usage.output_tokens, - ...extractPromptCacheTokens(usage), - }; - onUsageData(usageData); + onUsageData(toTokenUsage((messageStreamEvent as any).usage)); } } } catch (error) { From ae6893bc6498d45785fad095821813d13e21cce3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:38:55 -0700 Subject: [PATCH 60/81] refactor(guardrails): narrow the Responses tool-call item mapping types --- .../openai/responses/guardrail_translation/handler.py | 4 ++-- .../responses/test_openai_responses_guardrail_handler.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index e78e1f56915..fd0d7b66bee 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -193,11 +193,11 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES -def _tool_call_output_item_mapping(item: object) -> Mapping[str, Any] | None: +def _tool_call_output_item_mapping(item: object) -> Mapping[str, object] | None: if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES: return None if isinstance(item, Mapping): - return cast("Mapping[str, Any]", item) # cast-ok: output items are str-keyed JSON objects + return cast("Mapping[str, object]", item) # cast-ok: output items are str-keyed JSON objects return item.model_dump() if isinstance(item, BaseModel) else None diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 628ada05267..cf4895f637f 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -14,6 +14,7 @@ import pytest from fastapi import HTTPException +from pydantic import BaseModel from openai.types.responses import ( ResponseCustomToolCall, ResponseCustomToolCallInputDeltaEvent, @@ -28,6 +29,7 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.types.llms.openai import ChatCompletionToolCallChunk from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -68,7 +70,7 @@ class PersimmonMaskingGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[Any] = None, + logging_obj: Optional[LiteLLMLoggingObj] = None, ) -> GenericGuardrailAPIInputs: tool_calls = [ { @@ -593,7 +595,7 @@ class TestOpenAIResponsesHandlerToolCallExtraction: texts_to_check: List[str] = [] images_to_check: List[str] = [] - tool_calls_to_check: List[Any] = [] + tool_calls_to_check: List[ChatCompletionToolCallChunk] = [] task_mappings: List[Tuple[int, int]] = [] # Extract tool calls @@ -1476,7 +1478,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: ) handler = OpenAIResponsesHandler() - typed_events: List[Any] = [ + typed_events: List[BaseModel] = [ model.model_validate({**event, "sequence_number": sequence_number}) for sequence_number, (model, event) in enumerate( zip( From eee239cd9e6b3b3e4bc3d3a515fa7d5160b51df9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 15:20:58 -0700 Subject: [PATCH 61/81] fix(jwt): cascade-delete JWT key mappings when their virtual key is deleted LiteLLM_JWTKeyMapping_token_fkey was created ON DELETE RESTRICT, so deleting a virtual key that a JWT mapping pointed at failed with a foreign key violation on every deletion path (/key/delete, Admin UI, alias delete, team and user cascades). Declaring onDelete: Cascade on the relation lets the database clean the mapping up uniformly, so the next JWT call from that identity re-registers against the newly created key. Rebase of #33703 onto current staging. Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- .../migration.sql | 15 +++++ .../litellm_proxy_extras/schema.prisma | 2 +- litellm/proxy/schema.prisma | 2 +- schema.prisma | 2 +- .../test_litellm_proxy_extras_utils.py | 67 +++++++++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql new file mode 100644 index 00000000000..e5d48abcb52 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql @@ -0,0 +1,15 @@ +-- DropForeignKey +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_JWTKeyMapping_token_fkey') THEN + ALTER TABLE "LiteLLM_JWTKeyMapping" DROP CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey"; + END IF; +END $$; + +-- AddForeignKey +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_JWTKeyMapping_token_fkey') THEN + ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey" FOREIGN KEY ("token") REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE CASCADE ON UPDATE CASCADE; + END IF; +END $$; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/schema.prisma b/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/schema.prisma +++ b/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 3fab20a28ad..e5826b18668 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -2,6 +2,7 @@ import glob import os import re import sys +from pathlib import Path import pytest @@ -870,3 +871,69 @@ class TestMigrateDeployAttemptAccounting: harness.run() assert len(harness.deploy_calls) == 1 assert harness.resolved == [] + + +class TestJWTKeyMappingCascade: + """Regression tests for issue #33702. + + A virtual key referenced by a LiteLLM_JWTKeyMapping row could not be deleted + because LiteLLM_JWTKeyMapping_token_fkey was created ON DELETE RESTRICT, so + deleting the key (Admin UI, /key/delete, team delete, ...) raised a foreign + key violation. The mapping must be removed automatically when its key is + deleted, which the FK now enforces via ON DELETE CASCADE. + """ + + _FK_NAME = "LiteLLM_JWTKeyMapping_token_fkey" + + def _effective_on_delete(self): + """Replay every migration in order and return the last ON DELETE action + declared for the JWT key mapping FK.""" + action = None + for _migration_name, sql in _get_all_migrations(): + for match in re.finditer( + rf'ADD\s+CONSTRAINT\s+"{re.escape(self._FK_NAME)}".*?' + r"ON\s+DELETE\s+(CASCADE|RESTRICT|SET\s+NULL|NO\s+ACTION|SET\s+DEFAULT)", + sql, + re.IGNORECASE | re.DOTALL, + ): + action = re.sub(r"\s+", " ", match.group(1).upper()) + return action + + def test_fk_effective_on_delete_is_cascade(self): + """The final FK definition across all migrations must cascade deletes.""" + assert self._effective_on_delete() == "CASCADE", ( + f"{self._FK_NAME} must end up ON DELETE CASCADE so deleting a " + "virtual key removes its JWT key mapping (issue #33702)" + ) + + def test_schema_declares_cascade_on_relation(self): + """schema.prisma must declare onDelete: Cascade on the mapping relation + so the generated client and DB agree.""" + schema_paths = glob.glob( + os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../**/schema.prisma" + ) + ), + recursive=True, + ) + declaring = tuple( + (path, schema) + for path, schema in ((p, Path(p).read_text()) for p in schema_paths) + if "model LiteLLM_JWTKeyMapping" in schema + ) + assert declaring, "No schema.prisma declaring LiteLLM_JWTKeyMapping found" + for path, schema in declaring: + match = re.search( + r"litellm_verification_token\s+LiteLLM_VerificationToken\s+@relation\(([^)]*)\)", + schema, + ) + assert match is not None, ( + f"{path} declares LiteLLM_JWTKeyMapping but its verification token " + "relation could not be parsed, so this test cannot vouch for it " + "(issue #33702)" + ) + assert "onDelete: Cascade" in match.group(1), ( + f"{path} must declare onDelete: Cascade on the JWT key mapping " + "relation (issue #33702)" + ) From 6e96885629521b69c01c56faa165300963f6fb9a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:40:15 -0700 Subject: [PATCH 62/81] fix(proxy): accept non-string callback vars in default_team_settings (#40458) * fix(proxy): accept non-string callback vars in default_team_settings A YAML boolean such as turn_off_message_logging: true in a default_team_settings block failed TeamCallbackMetadata's str-only callback_vars validation and errored the request before any callback ran. Stringify the value the same way AddTeamCallback does. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop docstring from default_team_settings bool regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): move default_team_settings bool regression test to mapped pre_call_utils suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 4 ++- .../proxy/test_litellm_pre_call_utils.py | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2925226f235..da033dc2276 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1760,7 +1760,9 @@ class LiteLLMProxyRequestSetup: callback_vars_dict.pop("success_callback", None) callback_vars_dict.pop("failure_callback", None) callback_vars_dict = { - key: (litellm.utils.get_secret(value, default_value=value) or value if isinstance(value, str) else value) + key: ( + litellm.utils.get_secret(value, default_value=value) or value if isinstance(value, str) else str(value) + ) for key, value in callback_vars_dict.items() } diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 7564fa91c43..94aaa32519e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -35,6 +35,7 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY +from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_logging_from_dynamic_params from litellm.litellm_core_utils.get_provider_specific_headers import ( ProviderSpecificHeaderUtils, ) @@ -7909,3 +7910,36 @@ async def test_client_supplied_omit_marker_never_reaches_the_spend_log( if general_settings.get("missing_session_id") == "generate" else "per-call-random-trace-id" ) + + +def test_default_team_settings_bool_turn_off_message_logging_redacts(): + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc.config = { + "litellm_settings": { + "default_team_settings": [ + { + "team_id": "team-redact", + "success_callback": ["gcs_bucket"], + "failure_callback": ["gcs_bucket"], + "turn_off_message_logging": True, + } + ] + } + } + + callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config( + team_id="team-redact", + proxy_config=pc, + ) + + assert callback_metadata is not None + assert callback_metadata.success_callback == ["gcs_bucket"] + assert callback_metadata.callback_vars == {"turn_off_message_logging": "True"} + assert ( + _get_turn_off_message_logging_from_dynamic_params( + {"standard_callback_dynamic_params": dict(callback_metadata.callback_vars)} + ) + is True + ) From f5a410585af9fe2c06f0e5fa07ceddb2e0be46e0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:44:29 -0700 Subject: [PATCH 63/81] fix(responses): keep a custom tool_choice type in the bridged echo and type the new tests --- .../transformation.py | 23 +++++++++++++------ .../test_litellm_completion_responses.py | 17 +++++++++----- .../test_streaming_iterator_transformation.py | 9 ++++---- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 878f493b58e..6b9931b7df2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -27,9 +27,10 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( ) from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -272,13 +273,21 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_tool_choice_for_responses_api_response(tool_choice: object) -> ToolChoice: - normalized: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice) - match normalized: - case None: - return "auto" - case {"type": "function", "function": {"name": str(function_name)}}: + if tool_choice is None: + return "auto" + try: + return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(tool_choice) + except ValidationError: + return LiteLLMCompletionResponsesConfig._chat_tool_choice_as_responses_api_tool_choice(tool_choice) + + @staticmethod + def _chat_tool_choice_as_responses_api_tool_choice(tool_choice: object) -> ToolChoice: + match tool_choice, LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice): + case {"type": "custom"}, {"function": {"name": str(custom_name)}}: + return ToolChoiceCustomParam(type="custom", name=custom_name) + case _, {"type": "function", "function": {"name": str(function_name)}}: return ToolChoiceFunctionParam(type="function", name=function_name) - case _: + case _, normalized: return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) @staticmethod diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2aab660b5b7..8f2937cb252 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,4 +1,5 @@ import json +from typing import Final import pytest @@ -1426,7 +1427,9 @@ class TestToolChoiceTransformation: [ ({"type": "function", "name": "run_command"}, {"type": "function", "name": "run_command"}), ({"type": "function", "function": {"name": "run_command"}}, {"type": "function", "name": "run_command"}), - ({"type": "custom", "name": "ApplyPatch"}, {"type": "function", "name": "ApplyPatch"}), + ({"type": "custom", "name": "ApplyPatch"}, {"type": "custom", "name": "ApplyPatch"}), + ({"type": "custom", "custom": {"name": "ApplyPatch"}}, {"type": "custom", "name": "ApplyPatch"}), + ({"type": "function"}, "required"), ({"type": "tool"}, "required"), ({"type": "auto"}, "auto"), ("required", "required"), @@ -1434,14 +1437,16 @@ class TestToolChoiceTransformation: (None, "auto"), ], ) - def test_transform_tool_choice_for_responses_api_response(self, request_tool_choice, expected): - result = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + def test_transform_tool_choice_for_responses_api_response( + self, request_tool_choice: object, expected: str | dict[str, str] + ) -> None: + result: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( request_tool_choice ) assert result == expected - def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self): - chat_completion_response = ModelResponse( + def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self) -> None: + chat_completion_response: Final = ModelResponse( id="chatcmpl-named-tool-choice", created=1748575031, model="claude-haiku-4-5", @@ -1465,7 +1470,7 @@ class TestToolChoiceTransformation: ], ) - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + responses_api_response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( request_input="Run the command pwd.", responses_api_request={"tool_choice": {"type": "function", "name": "run_command"}}, chat_completion_response=chat_completion_response, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index d4b565f82a1..29061db97dc 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -11,6 +11,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve """ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -657,8 +658,8 @@ def _tool_call_chunk(finish_reason: str | None = None) -> ModelResponseStream: ) -def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): - iterator = LiteLLMCompletionStreamingIterator( +def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: + iterator: Final = LiteLLMCompletionStreamingIterator( model="claude-haiku-4-5", litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), request_input="Run the command pwd.", @@ -670,9 +671,9 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): litellm_metadata={}, ) - events = list(iterator) + events: Final = list(iterator) - response_events = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] + response_events: Final = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] assert [event.type for event in response_events] == [ "response.created", "response.in_progress", From eb45a088d3d3c255118d0effb41e4613af0907c0 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 15:53:05 -0700 Subject: [PATCH 64/81] fix(router): resolve team-scoped auto-routers by their public name (#40432) A team-scoped auto-router is stored under an internal model_name_{team_id}_{uuid} with the caller-facing name in model_info.team_public_model_name, and the four pre-routing strategy registries key on that internal name. A team key asks for the public name, so the strategy lookup missed, the team early-resolve exit handed back the marker deployment itself, and every call 400'd with "Unmapped LLM provider". The strategy lookup now resolves the requested name through the same team-first, then global, then admin-across-teams deployment resolution the deployment path uses, and looks the registries up under the model_name of whatever that resolves to. Both exits of _common_checks_available_deployment drop strategy markers through one helper, so a marker-only resolution is rejected as uncallable on every path. The request team id has one reader. Resolves LIT-7363 Claude-Session: https://claude.ai/code/session_01NU97S7d2FUDDvTk59k53Wp Co-authored-by: Claude Fable 5.1 --- .../guardrails/auto_router_compression.py | 25 +- litellm/router.py | 144 +++++---- litellm/router_utils/common_utils.py | 14 +- .../test_auto_router_compression.py | 26 +- .../proxy/test_common_request_processing.py | 2 +- .../router_strategy/test_complexity_router.py | 4 +- tests/test_litellm/test_router.py | 301 +++++++++++++++++- 7 files changed, 417 insertions(+), 99 deletions(-) diff --git a/litellm/proxy/guardrails/auto_router_compression.py b/litellm/proxy/guardrails/auto_router_compression.py index 98707e7ddca..c37b9fff1f0 100644 --- a/litellm/proxy/guardrails/auto_router_compression.py +++ b/litellm/proxy/guardrails/auto_router_compression.py @@ -80,17 +80,19 @@ def policy_from_litellm_params(litellm_params: Mapping[str, object]) -> AutoRout def policy_for_model( llm_router: "Router | None", model_alias: str, - team_id: str | None, + request_kwargs: Mapping[str, object], request_tags: Sequence[str], ) -> AutoRouterCompressionPolicy | None: - """The compression policy of the auto router marker `model_alias` resolves to. + """The compression policy of the auto router marker `model_alias` resolves to for this caller. - Pre-call arming and the routing hook both resolve through here, so an alias with - several tag-scoped markers cannot suppress under one and then route under another. + Pre-call arming and the routing hook both resolve through here, and here resolves through the + router's own request-scoped deployment lookup, so an alias with several tag-scoped markers + cannot suppress under one and then route under another, and a team router reached by its + public name carries its policy for every principal that can reach it. """ if llm_router is None: return None - deployments: Final = llm_router.get_model_list(model_name=model_alias, team_id=team_id) or () + deployments: Final = llm_router.deployments_for_request(model_alias, request_kwargs) markers: Final = tuple( litellm_params for deployment in deployments @@ -108,17 +110,6 @@ def policy_for_model( return next((policy for policy in candidates if policy is not None), None) -def team_id_from_request(request_kwargs: Mapping[str, object]) -> str | None: - """The caller's team id, from whichever metadata bucket this surface writes to.""" - for meta_key in ("metadata", "litellm_metadata"): - meta = request_kwargs.get(meta_key) - if isinstance(meta, Mapping): - team_id = meta.get("user_api_key_team_id") - if isinstance(team_id, str): - return team_id - return None - - def _compression_guardrail_classes() -> tuple[type, ...]: """The registered guardrail classes whose provider compresses prompts.""" from litellm.proxy.guardrails.guardrail_registry import guardrail_class_registry @@ -172,7 +163,7 @@ async def arm_pre_call( policy: Final = policy_for_model( llm_router=llm_router, model_alias=model_alias, - team_id=team_id_from_request(data), + request_kwargs=data, request_tags=_get_tags_from_request_kwargs(data), ) if policy is None: diff --git a/litellm/router.py b/litellm/router.py index 68ace283949..88601e5b97b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -148,6 +148,7 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + get_request_team_id, resolve_model_group_alias, truncate_fallback_error_detail, warn_on_provider_credential_mismatch, @@ -12337,27 +12338,7 @@ class Router: if team_deployments: return model, team_deployments elif include_team_models: - team_deployments = [ - self.model_list[index] - for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() - if public_model_name == model - for index in indices - ] - team_ids: Final = { - team_id - for deployment in team_deployments - for team_id in [(deployment.get("model_info") or {}).get("team_id")] - if team_id is not None - } - if len(team_ids) > 1: - raise litellm.BadRequestError( - message=( - f"Model name '{model}' matches deployments from multiple teams. " - "Specify the deployment ID directly to disambiguate." - ), - model=model, - llm_provider="", - ) + team_deployments = self._team_deployments_across_teams(model) if team_deployments: return model, team_deployments @@ -12384,6 +12365,45 @@ class Router: return None + def _team_deployments_across_teams(self, model: str) -> list[DeploymentTypedDict]: + """Every team's deployments under public name `model`, for a proxy admin calling without a team.""" + team_deployments: Final = [ + self.model_list[index] + for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() + if public_model_name == model + for index in indices + ] + team_ids: Final = { + team_id + for deployment in team_deployments + for team_id in [(deployment.get("model_info") or {}).get("team_id")] + if team_id is not None + } + if len(team_ids) > 1: + raise litellm.BadRequestError( + message=( + f"Model name '{model}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=model, + llm_provider="", + ) + return team_deployments + + def deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[DeploymentTypedDict]: + """The deployments `model` names for this caller, through the same alias, then team-first, then + global, then admin-across-teams resolution `_common_checks_available_deployment` applies, so + strategy selection and compression policy can never disagree with deployment selection about + which marker a name means.""" + registered_name: Final = self._get_model_from_alias(model=model) or model + team_id: Final = get_request_team_id(request_kwargs) + deployments: Final = self._get_all_deployments(model_name=registered_name, team_id=team_id) + if deployments or team_id is not None or not _is_proxy_admin_request(request_kwargs): + return deployments + return self._team_deployments_across_teams(registered_name) + @staticmethod def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: litellm_params: Final = deployment.get("litellm_params") @@ -12411,11 +12431,7 @@ class Router: - Dict, if specific model chosen """ - request_team_id: str | None = None - if request_kwargs is not None: - metadata: Final = request_kwargs.get("metadata") or {} - litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {} - request_team_id = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id") + request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: return model, self._get_deployment_by_litellm_model(model=model) @@ -12440,7 +12456,9 @@ class Router: include_team_models=_is_proxy_admin_request(request_kwargs), ) if early is not None: - return early + if not isinstance(early[1], list): + return early + return early[0], self._drop_strategy_markers(early[0], early[1]) ## get healthy deployments ### get all deployments @@ -12517,19 +12535,22 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) - if not any(marker_flags): - return model, healthy_deployments - selectable: Final = [ # mutable-ok: matches this function's list contract expected by downstream filters - d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + return model, self._drop_strategy_markers(model, healthy_deployments) + + def _drop_strategy_markers( + self, model: str, deployments: Sequence[DeploymentTypedDict] + ) -> list[DeploymentTypedDict]: + """A strategy marker is never a callable deployment, whichever resolution arm produced it.""" + selectable: Final = [ # mutable-ok: matches _common_checks_available_deployment's list contract + d for d in deployments if not self._is_strategy_marker_deployment(d) ] - if not selectable: + if deployments and not selectable: raise litellm.BadRequestError( message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}", model=model, llm_provider="", ) - return model, selectable + return selectable def _filter_deployments_by_model_access_groups( self, @@ -13219,12 +13240,8 @@ class Router: return filtered - def _model_name_has_plain_deployments(self, model: str) -> bool: - indices: Final = self.model_name_to_deployment_indices.get(model) or () - return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) - def _select_pre_routing_strategy( - self, model: str, request_kwargs: dict + self, model: str, request_kwargs: Mapping[str, object] ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments @@ -13235,6 +13252,12 @@ class Router: deployment the strategy was registered from via its (model_name, tags) pair. + The registries are keyed by the marker deployment's own `model_name`, which + for a team-scoped router is the internal `model_name_{team}_{uuid}` while + the caller sends the team's public name. So the names looked up are the + `model_name`s of whatever deployments this caller's request resolves `model` + to, and `model` itself when it resolves to none. + With tag filtering enabled, router-wide or by the request's enable_tag_filtering (which the proxy sets from key/team router_settings), strategies that all carry real tags matching none of @@ -13242,12 +13265,14 @@ class Router: deployments: returning None hands the request to ordinary tag-aware deployment selection. """ - candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ - *self.auto_routers.get(model, []), - *self.complexity_routers.get(model, []), - *self.adaptive_routers.get(model, []), - *self.quality_routers.get(model, []), - ] + registries: Final = (self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers) + if not any(registries): + return None + deployments: Final = self.deployments_for_request(model, request_kwargs) + registered_names: Final = tuple(dict.fromkeys(str(d["model_name"]) for d in deployments)) or (model,) + candidates: Final = tuple( + tagged for registry in registries for name in registered_names for tagged in registry.get(name, []) + ) if not candidates: return None @@ -13265,7 +13290,7 @@ class Router: if ( (self.enable_tag_filtering or request_scoped_filtering) and all(tagged.tags for tagged in candidates) - and self._model_name_has_plain_deployments(model) + and any(not self._is_strategy_marker_deployment(d) for d in deployments) ): return None return candidates[0] @@ -13377,11 +13402,12 @@ class Router: Used for the litellm auto-router to modify the request before the routing decision is made. - `model` is whatever the caller asked for, which may be a `model_group_alias` key, while the - strategy registries and the marker deployment are keyed by the marker's own `model_name`, so - every lookup below resolves the alias first. Only the lookups: the caller-facing name stays - the alias, since spend metadata is stamped before routing and the response carries the tier - group the strategy picked. + `model` is whatever the caller asked for, which may be a `model_group_alias` key or a team's + public model name, while the strategy registries and the marker deployment are keyed by the + marker's own `model_name`, so every lookup below resolves the alias first and the team name + through the deployment path. Only the lookups: the caller-facing name stays the alias, since + spend metadata is stamped before routing and the response carries the tier group the + strategy picked. """ requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model registered_model_name: Final = await self._resolve_claude_code_session_router( @@ -13418,7 +13444,6 @@ class Router: messages_for_routing, model_hop_compression_armed, policy_for_model, - team_id_from_request, ) # Same tag-aware lookup the proxy's pre-call arming used, so an alias with @@ -13426,7 +13451,7 @@ class Router: compression_policy: Final = policy_for_model( llm_router=self, model_alias=registered_model_name, - team_id=team_id_from_request(request_kwargs), + request_kwargs=request_kwargs, request_tags=_get_tags_from_request_kwargs(request_kwargs), ) # Shared compression already ran in the pre-call hook, so reuse it rather than @@ -13495,7 +13520,9 @@ class Router: # Per-tier `litellm_params` on the hook response are deliberate overrides # the caller applies on top, so those keys are never forwarded here. marker_params: Final = ( - self._forwardable_alias_marker_params(model=registered_model_name, strategy_tags=selected_strategy.tags) + self._forwardable_alias_marker_params( + model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs + ) if pre_routing_hook_response is not None else () ) @@ -13513,13 +13540,14 @@ class Router: return pre_routing_hook_response def _forwardable_alias_marker_params( - self, model: str, strategy_tags: tuple[str, ...] + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] ) -> tuple[tuple[str, object], ...]: marker_params: Final = tuple( litellm_params - for idx in self.model_name_to_deployment_indices.get(model, ()) - if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict) - and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + for deployment in self.deployments_for_request(model, request_kwargs) + if str((litellm_params := deployment["litellm_params"]).get("model", "")).startswith( + AUTO_ROUTER_MODEL_PREFIX + ) ) tag_matched: Final = tuple( params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 280a7defcf8..c1c6d25beca 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -26,6 +26,18 @@ def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool return getattr(user_api_key_auth, "user_role", None) == "proxy_admin" +def get_request_team_id(request_kwargs: Mapping[str, object] | None) -> str | None: + """The caller's team id, from whichever metadata bucket this surface writes to.""" + if request_kwargs is None: + return None + for bucket_name in ("metadata", "litellm_metadata"): + bucket = request_kwargs.get(bucket_name) + team_id = bucket.get("user_api_key_team_id") if isinstance(bucket, Mapping) else None + if isinstance(team_id, str) and team_id: + return team_id + return None + + def resolve_model_group_alias(model_group_alias: object, model: str) -> str | None: """ Resolve ``model`` through a ``model_group_alias`` map. @@ -110,7 +122,7 @@ def filter_team_based_models( metadata: Final = request_kwargs.get("metadata") or {} litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {} - request_team_id: Final = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id") + request_team_id: Final = get_request_team_id(request_kwargs) if request_team_id is None and _is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list): requested_model: Final = ( request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group") diff --git a/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py b/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py index db2f94306fb..0d723d6671f 100644 --- a/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py +++ b/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py @@ -56,13 +56,13 @@ class TestPolicyFromLitellmParams: class _FakeRouter: - """Minimal stand-in for litellm.Router.get_model_list, for policy_for_model.""" + """Minimal stand-in for litellm.Router.deployments_for_request, for policy_for_model.""" def __init__(self, deployments: list[dict[str, Any]]): self._deployments = deployments - def get_model_list(self, model_name, team_id=None): - return [d for d in self._deployments if d.get("model_name") == model_name] + def deployments_for_request(self, model, request_kwargs): + return [d for d in self._deployments if d.get("model_name") == model] def _marker(compression: dict[str, str], tags: list[str] | None = None) -> dict[str, Any]: @@ -78,23 +78,23 @@ def _marker(compression: dict[str, str], tags: list[str] | None = None) -> dict[ class TestPolicyForModel: def test_no_router_returns_none(self): - assert policy_for_model(llm_router=None, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=None, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_no_marker_deployment_returns_none(self): router = _FakeRouter([{"model_name": "smart-router", "litellm_params": {"model": "openai/gpt-4o-mini"}}]) - assert policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_marker_deployment_without_policy_returns_none(self): router = _FakeRouter( [{"model_name": "smart-router", "litellm_params": {"model": "auto_router/complexity_router"}}] ) - assert policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_marker_deployment_with_policy_is_found(self): router = _FakeRouter( [_marker({"auto_router_routing_compression": "headroom-a", "auto_router_model_compression": "none"})] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) assert policy == AutoRouterCompressionPolicy(routing="headroom-a", model=None) def test_picks_the_marker_whose_tags_the_request_carries(self): @@ -107,8 +107,8 @@ class TestPolicyForModel: ] ) - eu = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("eu",)) - us = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) + eu = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("eu",)) + us = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) assert eu == AutoRouterCompressionPolicy(routing="headroom-eu", model=None) assert us == AutoRouterCompressionPolicy(routing="headroom-us", model=None) @@ -116,7 +116,7 @@ class TestPolicyForModel: def test_untagged_marker_matches_any_request(self): router = _FakeRouter([_marker({"auto_router_routing_compression": "headroom-a"})]) policy = policy_for_model( - llm_router=router, model_alias="smart-router", team_id=None, request_tags=("anything",) + llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("anything",) ) assert policy == AutoRouterCompressionPolicy(routing="headroom-a", model=None) @@ -128,14 +128,14 @@ class TestPolicyForModel: _marker({"auto_router_routing_compression": "headroom-default"}), ] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) assert policy == AutoRouterCompressionPolicy(routing="headroom-default", model=None) def test_no_untagged_fallback_means_no_policy(self): """No matching marker means no policy, not an unrelated slice's compression.""" router = _FakeRouter([_marker({"auto_router_routing_compression": "headroom-eu"}, tags=["eu"])]) assert ( - policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) is None + policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) is None ) def test_tag_scoped_marker_takes_precedence_over_untagged(self): @@ -147,7 +147,7 @@ class TestPolicyForModel: _marker({"auto_router_routing_compression": "headroom-eu"}, tags=["eu"]), ] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("eu",)) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("eu",)) assert policy == AutoRouterCompressionPolicy(routing="headroom-eu", model=None) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index d2ba6527826..50b26577e5c 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -417,7 +417,7 @@ class TestProxyBaseLLMRequestProcessing: ) fake_llm_router = MagicMock() - fake_llm_router.get_model_list.return_value = [ + fake_llm_router.deployments_for_request.return_value = [ { "model_name": "smart-router", "litellm_params": { diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 9f925499f1a..51103297c58 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -3846,11 +3846,11 @@ class TestRouterPreRoutingSharedAliasName: def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self): router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()]) - forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=())) + forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=(), request_kwargs={})) assert forwarded["drop_params"] is True assert "api_key" not in forwarded and "api_base" not in forwarded - assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=(), request_kwargs={}) == () @staticmethod def _region_marker_entry() -> dict: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 80da922724d..1e0327e5ac2 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10003,13 +10003,6 @@ class TestTaggedAutoRouterOnSharedModelName: def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False - def test_model_name_has_plain_deployments_reflects_the_pool(self): - mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) - marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) - - assert mixed._model_name_has_plain_deployments("gpt4o") is True - assert marker_only._model_name_has_plain_deployments("gpt4o") is False - class TestAutoRouterSharedModelNameConnectionParams: """A plain deployment sharing its model_name with an `auto_router/` marker must not have @@ -10618,6 +10611,300 @@ class TestModelGroupAliasReachesPreRoutingStrategies: ) +class TestTeamPublicNameReachesPreRoutingStrategies: + """A team-scoped strategy router is stored under an internal `model_name_{team}_{uuid}` with the + caller-facing name in `model_info.team_public_model_name`, and the four registries key on that + internal name. A team key asks for the public name, so the hook has to resolve it to the team's + marker through the same team-first resolution the deployment path uses, and a resolution that + yields only markers is not callable on any path (LIT-7363).""" + + MARKER_TIMEOUT = 42.0 + REGISTRY_NAMES = ("auto_routers", "complexity_routers", "adaptive_routers", "quality_routers") + TEAM = "team-a" + OTHER_TEAM = "team-b" + PUBLIC_NAME = "smart-route" + INTERNAL_NAME = "model_name_team-a_0b3c" + SIBLING_INTERNAL_NAME = "model_name_team-a_9e1d" + + class _RewriteStrategy: + def __init__(self, rewrite_to: str = "gemini-flash"): + self.rewrite_to = rewrite_to + + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse(model=self.rewrite_to, messages=messages) + + @classmethod + def _team_marker(cls, internal_name: str, tags: list[str] | None = None) -> dict: + tiers = dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), "gemini-flash") + return { + "model_name": internal_name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "gemini-flash", + "timeout": cls.MARKER_TIMEOUT, + **({"tags": tags} if tags else {}), + }, + "model_info": {"team_id": cls.TEAM, "team_public_model_name": cls.PUBLIC_NAME}, + } + + @classmethod + def _router( + cls, + registrations: dict[str, "TestTeamPublicNameReachesPreRoutingStrategies._RewriteStrategy"], + registry_name: str = "complexity_routers", + extra_deployments: tuple[dict, ...] = (), + markers: tuple[dict, ...] | None = None, + enable_tag_filtering: bool = False, + ) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + markers = markers if markers is not None else (cls._team_marker(cls.INTERNAL_NAME),) + tier = { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "mock_response": "routed by the tier"}, + } + router = litellm.Router( + model_list=[*markers, tier, *extra_deployments], + enable_tag_filtering=enable_tag_filtering, + ) + tags_by_name = {m["model_name"]: tuple(m["litellm_params"].get("tags") or ()) for m in markers} + for name in cls.REGISTRY_NAMES: + setattr(router, name, {}) + setattr( + router, + registry_name, + { + name: [TaggedPreRoutingStrategy(tags=tags_by_name[name], strategy=strategy)] + for name, strategy in registrations.items() + }, + ) + return router + + @staticmethod + def _messages() -> list[dict[str, str]]: + return [{"role": "user", "content": "What is the capital of France?"}] + + @classmethod + def _team_request(cls, team_id: str | None = "team-a", tags: list[str] | None = None) -> dict: + metadata = {**({"user_api_key_team_id": team_id} if team_id else {}), **({"tags": tags} if tags else {})} + return {"metadata": metadata} + + @pytest.mark.parametrize("registry_name", REGISTRY_NAMES) + @pytest.mark.asyncio + async def test_team_key_dispatches_to_the_strategy_registered_under_the_internal_name(self, registry_name): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}, registry_name=registry_name) + + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_team_key_deployment_selection_lands_on_the_tier_and_forwards_the_marker_params(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + request_kwargs = self._team_request() + + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + + assert deployment["litellm_params"]["model"] == "gemini/gemini-3.6-flash" + assert request_kwargs["timeout"] == self.MARKER_TIMEOUT + + @pytest.mark.asyncio + async def test_another_team_never_reaches_the_strategy_or_the_marker(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + + assert ( + await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(self.OTHER_TEAM), messages=self._messages() + ) + is None + ) + with pytest.raises(litellm.BadRequestError): + await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(self.OTHER_TEAM), messages=self._messages() + ) + + @pytest.mark.asyncio + async def test_sibling_team_markers_select_by_request_tag_then_default(self): + router = self._router( + { + self.INTERNAL_NAME: self._RewriteStrategy("cn-model"), + self.SIBLING_INTERNAL_NAME: self._RewriteStrategy("us-model"), + }, + markers=( + self._team_marker(self.INTERNAL_NAME, tags=["cn"]), + self._team_marker(self.SIBLING_INTERNAL_NAME, tags=["us", "default"]), + ), + ) + + async def routed(tags: list[str] | None) -> str | None: + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(tags=tags), messages=self._messages() + ) + return response.model if response else None + + assert await routed(["cn"]) == "cn-model" + assert await routed(["us"]) == "us-model" + assert await routed(None) == "us-model" + + @pytest.mark.asyncio + async def test_team_public_name_shadows_a_global_model_for_that_team_only(self): + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + extra_deployments=({"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},), + ) + + async def routed(request_kwargs: dict) -> str | None: + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + return response.model if response else None + + async def selected(request_kwargs: dict) -> str: + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + return deployment["litellm_params"]["model"] + + assert await routed(self._team_request()) == "gemini-flash" + assert await selected(self._team_request()) == "gemini/gemini-3.6-flash" + for request_kwargs in (self._team_request(None), self._team_request(self.OTHER_TEAM)): + assert await routed(request_kwargs) is None + assert await selected(request_kwargs) == "openai/gpt-4o" + + @pytest.mark.asyncio + async def test_tag_filtering_hands_untagged_team_requests_to_the_team_plain_sibling(self): + plain_sibling = { + "model_name": self.SIBLING_INTERNAL_NAME, + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"team_id": self.TEAM, "team_public_model_name": self.PUBLIC_NAME}, + } + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + markers=(self._team_marker(self.INTERNAL_NAME, tags=["route"]),), + extra_deployments=(plain_sibling,), + enable_tag_filtering=True, + ) + + tagged = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(tags=["route"]), messages=self._messages() + ) + assert tagged is not None and tagged.model == "gemini-flash" + for _ in range(20): + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + + @pytest.mark.asyncio + async def test_marker_only_team_resolution_is_rejected_as_uncallable(self): + import re + + from litellm.types.router import RouterErrors + + router = self._router({}) + + with pytest.raises( + litellm.BadRequestError, match=re.escape(RouterErrors.only_strategy_marker_deployments.value) + ): + await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + + @pytest.mark.asyncio + async def test_proxy_admin_without_a_team_reaches_the_team_strategy_by_public_name(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + request_kwargs = {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}} + + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + + assert response is not None + assert response.model == "gemini-flash" + @pytest.mark.asyncio + async def test_strategy_resolution_agrees_with_the_deployment_path_for_every_principal(self): + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + extra_deployments=({"model_name": "shared-name", "litellm_params": {"model": "openai/gpt-4o"}},), + ) + principals = { + "team": self._team_request(), + "other-team": self._team_request(self.OTHER_TEAM), + "no-team": self._team_request(None), + "admin": {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}}, + } + for principal, request_kwargs in principals.items(): + for model in (self.PUBLIC_NAME, "shared-name", "gemini-flash", "missing"): + resolved = [d["model_name"] for d in router.deployments_for_request(model, request_kwargs)] + callable_names = [ + name + for name, deployment in zip(resolved, router.deployments_for_request(model, request_kwargs)) + if not router._is_strategy_marker_deployment(deployment) + ] + if resolved and not callable_names: + with pytest.raises(litellm.BadRequestError, match="strategy router marker"): + router._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) + elif not resolved: + with pytest.raises(litellm.BadRequestError): + router._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) + else: + _, deployments = router._common_checks_available_deployment( + model=model, request_kwargs=request_kwargs + ) + assert [d["model_name"] for d in deployments] == callable_names, (principal, model) + + def test_drop_strategy_markers_keeps_plain_deployments_and_rejects_marker_only_sets(self): + router = self._router({}) + marker = router.model_list[0] + plain = {"model_name": "plain", "litellm_params": {"model": "openai/gpt-4o"}} + + assert router._drop_strategy_markers("x", [marker, plain]) == [plain] + assert router._drop_strategy_markers("x", [plain]) == [plain] + assert router._drop_strategy_markers("x", []) == [] + with pytest.raises(litellm.BadRequestError, match="strategy router marker"): + router._drop_strategy_markers("x", [marker]) + + def test_team_deployments_across_teams_unions_one_team_and_rejects_two(self): + other_team_marker = { + **self._team_marker(self.SIBLING_INTERNAL_NAME), + "model_info": {"team_id": self.OTHER_TEAM, "team_public_model_name": self.PUBLIC_NAME}, + } + one_team = self._router({}) + two_teams = self._router({}, markers=(self._team_marker(self.INTERNAL_NAME), other_team_marker)) + + assert [d["model_name"] for d in one_team._team_deployments_across_teams(self.PUBLIC_NAME)] == [ + self.INTERNAL_NAME + ] + assert one_team._team_deployments_across_teams("missing") == [] + with pytest.raises(litellm.BadRequestError, match="multiple teams"): + two_teams._team_deployments_across_teams(self.PUBLIC_NAME) + + + def test_compression_policy_follows_the_same_resolution_for_every_principal(self): + from litellm.proxy.guardrails.auto_router_compression import AutoRouterCompressionPolicy, policy_for_model + + marker = self._team_marker(self.INTERNAL_NAME) + marker["litellm_params"]["auto_router_routing_compression"] = "headroom-team" + router = self._router({}, markers=(marker,)) + admin = {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}} + expected = AutoRouterCompressionPolicy(routing="headroom-team", model=None) + + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(), ()) == expected + assert policy_for_model(router, self.PUBLIC_NAME, admin, ()) == expected + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(self.OTHER_TEAM), ()) is None + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(None), ()) is None + + class TestAutoRouterCompressionDecoupling: """An auto router's `auto_router_routing_compression` / `auto_router_model_compression` decouple what the routing decision sees from what the model call sees. The one From fc161faa96a27f16d90d1aee6e4fc0574f679dd9 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 9 Sep 2026 15:56:38 -0700 Subject: [PATCH 65/81] fix(langfuse): give each call in a session header its own trace instead of upserting one trace per session (#40177) * fix(langfuse): give session-header calls their own trace id A client that sends only a session header (x-litellm-session-id, a vendor x--session-id such as Claude Code's X-Claude-Code-Session-Id, a bare x-session-id, or the Codex session/thread/conversation family) has that value stamped into both trace_id and session_id by the proxy. Langfuse upserts a trace by id, so every turn of a session collapsed into one growing trace and the Sessions view showed "Total traces: 1" Detect that aliasing in the Langfuse callback from the request headers the callback already receives, and use litellm_call_id as the trace id for those calls. session_id still carries the header value, so the turns stay grouped under one session. An explicit x-litellm-trace-id, langfuse_trace_id, or langfuse_existing_trace_id keeps its trace id, including when the caller sets it to the same value as the session id Co-authored-by: jesus * test(langfuse): cover direct-SDK callers without proxy request headers * fix(langfuse): preserve session trace provenance --------- Co-authored-by: jesus --- litellm/integrations/langfuse/langfuse.py | 59 +++- .../integrations/test_langfuse.py | 251 ++++++++++++++++++ 2 files changed, 306 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 9576eabaa34..b75369965de 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -2,6 +2,7 @@ # On success, logs events to Langfuse import inspect import os +import re import traceback from collections.abc import Callable, Iterable, Mapping from datetime import datetime @@ -63,6 +64,44 @@ def _object_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, dict) else None +def _widened_items(mapping: Mapping[str, object]) -> Iterable[tuple[object, object]]: + """Header pairs with the key type widened back to what a caller-supplied dict can actually hold.""" + return mapping.items() + + +def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_request: object) -> bool: + if not isinstance(trace_id, str) or not isinstance(session_id, str): + return False + request: Final = _object_mapping(proxy_server_request) + raw_headers: Final = _object_mapping(request.get("headers")) if request is not None else None + if raw_headers is None: + return False + headers: Final = MappingProxyType( + {key.lower(): value for key, value in _widened_items(raw_headers) if isinstance(key, str)} + ) + if headers.get("x-litellm-trace-id"): + return False + if headers.get("langfuse_trace_id") is not None: + return False + if trace_id != session_id and headers.get("langfuse_session_id") != session_id: + return False + if headers.get("x-litellm-session-id") == trace_id: + return True + if re.fullmatch(r"[a-zA-Z0-9_\-]{8,}", trace_id) is None: + return False + user_agent: Final = headers.get("user-agent") + codex: Final = isinstance(user_agent, str) and re.match(r"^codex[-_ /]", user_agent, re.IGNORECASE) is not None + return any( + value == trace_id + and ( + key == "x-session-id" + or re.fullmatch(r"x-.+-session-id", key) is not None + or (codex and key in ("session-id", "session_id", "thread-id", "conversation_id")) + ) + for key, value in headers.items() + ) + + class _UsageObject(Protocol): """Token-count surface the Langfuse logger reads off a response usage payload.""" @@ -609,6 +648,18 @@ class LangFuseLogger: # This allows continuing an existing trace while still returning the correct trace_id if existing_trace_id is not None: trace_id = existing_trace_id + resolved_trace_id: Final = ( + litellm_call_id or trace_id + if existing_trace_id is None + and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request")) + else trace_id + ) + if resolved_trace_id != trace_id: + verbose_logger.debug( + "Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace", + trace_id, + resolved_trace_id, + ) requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ())) update_trace_keys: Final = ( requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else () @@ -663,7 +714,7 @@ class LangFuseLogger: trace_params["output"] = masked_output if not mask_output else "redacted-by-litellm" else: # don't overwrite an existing trace trace_params = { - "id": trace_id, + "id": resolved_trace_id, "name": trace_name, "session_id": session_id, "input": masked_input if not mask_input else "redacted-by-litellm", @@ -845,13 +896,13 @@ class LangFuseLogger: # Verify langfuse accepted our trace_id; if it differs, log a warning but still return our intended value # to match expected test behavior if hasattr(generation_client, "trace_id") and generation_client.trace_id: - if generation_client.trace_id != trace_id: + if generation_client.trace_id != resolved_trace_id: verbose_logger.warning( "Langfuse trace_id mismatch: set %s, but langfuse returned %s. Using our intended trace_id for consistency.", - trace_id, + resolved_trace_id, generation_client.trace_id, ) - return trace_id, generation_id + return resolved_trace_id, generation_id except Exception: verbose_logger.error("Langfuse Layer Error - %s", traceback.format_exc()) return None, None diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index d36878e455f..87e76499b84 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -1341,6 +1341,257 @@ def _emit(logger: LangFuseLogger, *, metadata=None, headers=None): ) +@pytest.mark.parametrize("level", ["DEFAULT", "ERROR"]) +@pytest.mark.parametrize( + "headers,metadata,expected_id", + [ + ({"x-litellm-session-id": "session-7125"}, {}, "call"), + ({"X-Claude-Code-Session-Id": "session-7125"}, {}, "call"), + ({"x-session-id": "session-7125"}, {}, "call"), + ({"session-id": "session-7125", "user-agent": "codex_cli_rs/1.0"}, {}, "call"), + ({"thread-id": "session-7125", "user-agent": "codex-tui"}, {}, "call"), + ({"session_id": "session-7125", "user-agent": "Codex 1.0"}, {}, "call"), + ({"conversation_id": "session-7125", "user-agent": "codex_vscode/1.0"}, {}, "call"), + ({"x-litellm-session-id": "short"}, {}, "call"), + ({"x-litellm-trace-id": "session-7125"}, {}, "session-7125"), + ( + {"X-LiteLLM-Trace-Id": "session-7125", "x-litellm-session-id": "session-7125"}, + {}, + "session-7125", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_trace_id": "session-7125"}, + {}, + "session-7125", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_trace_id": "explicit-trace"}, + {}, + "explicit-trace", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_existing_trace_id": "existing-trace"}, + {}, + "existing-trace", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"x-litellm-session-id": "short", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"X-Claude-Code-Session-Id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"x-session-id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + { + "session-id": "session-7125", + "user-agent": "codex_cli_rs/1.0", + "langfuse_session_id": "custom-session", + }, + {}, + "call", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "x-litellm-trace-id": "explicit-trace", + }, + {}, + "explicit-trace", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "langfuse_trace_id": "explicit-trace", + }, + {}, + "explicit-trace", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "langfuse_existing_trace_id": "existing-trace", + }, + {}, + "existing-trace", + ), + ({}, {"trace_id": "session-7125", "session_id": "session-7125"}, "session-7125"), + ({}, {"trace_id": "explicit-trace", "session_id": "session-7125"}, "explicit-trace"), + ( + {"x-vendor-session-id": "short"}, + {"trace_id": "short", "session_id": "short"}, + "short", + ), + ( + {"x-session-id": "invalid value"}, + {"trace_id": "invalid value", "session_id": "invalid value"}, + "invalid value", + ), + ( + {"session-id": "session-7125", "user-agent": "codexfoo/1.0"}, + {"trace_id": "session-7125", "session_id": "session-7125"}, + "session-7125", + ), + ( + {"x-vendor-session-id": "short"}, + {"trace_id": "session-7125", "session_id": "session-7125"}, + "session-7125", + ), + ({}, {}, "call"), + ], +) +def test_session_header_trace_provenance(headers, metadata, expected_id, level): + from starlette.datastructures import Headers + + from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + clean_headers, + redact_credential_headers, + ) + + logger: Final = _steering_logger() + for turn in range(2): + call_id = f"call-{turn}" + request_headers = Headers(headers) + data = LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=request_headers, data={"metadata": dict(metadata)}, _metadata_variable_name="metadata" + ) + original_metadata = dict(data["metadata"]) + now = datetime.datetime.now() + result = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": call_id, + "litellm_trace_id": data.get("litellm_trace_id"), + "litellm_params": { + "metadata": data["metadata"], + "proxy_server_request": {"headers": redact_credential_headers(clean_headers(request_headers))}, + }, + "messages": [{"role": "user", "content": f"turn {turn}"}], + "optional_params": {}, + }, + response_obj=( + None + if level == "ERROR" + else litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]) + ), + start_time=now, + end_time=now, + level=level, + status_message="provider error" if level == "ERROR" else None, + ) + trace_params = logger.Langfuse.trace.call_args.kwargs + assert trace_params["id"] == (call_id if expected_id == "call" else expected_id) + assert result["trace_id"] == trace_params["id"] + if expected_id != "existing-trace": + assert trace_params["session_id"] == headers.get("langfuse_session_id", original_metadata.get("session_id")) + steering = {key[len("langfuse_") :]: value for key, value in headers.items() if key.startswith("langfuse_")} + assert data["metadata"] == {**original_metadata, **steering} + + +def test_session_header_trace_without_call_id_keeps_session_alias(): + logger: Final = _steering_logger() + now: Final = datetime.datetime.now() + + result: Final = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": "", + "litellm_params": { + "metadata": {"trace_id": "session-7125", "session_id": "session-7125"}, + "proxy_server_request": {"headers": {"x-litellm-session-id": "session-7125"}}, + }, + "messages": [{"role": "user", "content": "no call id"}], + "optional_params": {}, + }, + response_obj=litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]), + start_time=now, + end_time=now, + ) + + assert logger.Langfuse.trace.call_args.kwargs["id"] == "session-7125" + assert result["trace_id"] == "session-7125" + + +def test_every_proxy_session_header_shape_is_classified_as_a_session_alias(): + """The classifier must cover every header shape the proxy turns into a chain id.""" + from litellm.integrations.langfuse.langfuse import _is_session_header_trace + from litellm.proxy.litellm_pre_call_utils import ( + _CODEX_SESSION_ID_HEADERS, + get_chain_id_from_headers, + ) + + session: Final = "session-7125-abcdef" + session_shapes: Final = ( + {"x-litellm-session-id": session}, + {"X-Claude-Code-Session-Id": session}, + {"x-session-id": session}, + *({header: session, "user-agent": "codex_cli_rs/1.0"} for header in _CODEX_SESSION_ID_HEADERS), + ) + for headers in session_shapes: + assert get_chain_id_from_headers(dict(headers)) == session, headers + assert _is_session_header_trace(session, session, {"headers": headers}) is True, headers + + explicit_trace: Final = {"x-litellm-trace-id": session, "x-litellm-session-id": session} + assert get_chain_id_from_headers(dict(explicit_trace)) == session + assert _is_session_header_trace(session, session, {"headers": explicit_trace}) is False + + +@pytest.mark.parametrize( + "proxy_server_request", + [None, {}, {"headers": None}], + ids=["no-proxy-request", "no-headers-key", "null-headers"], +) +def test_sdk_caller_without_request_headers_keeps_its_trace(proxy_server_request): + """A direct SDK caller has no request headers, so a session-shaped trace id stays the caller's.""" + logger: Final = _steering_logger() + now: Final = datetime.datetime.now() + + result: Final = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": "call-0", + "litellm_params": { + "metadata": {"trace_id": "session-7125", "session_id": "session-7125"}, + "proxy_server_request": proxy_server_request, + }, + "messages": [{"role": "user", "content": "sdk turn"}], + "optional_params": {}, + }, + response_obj=litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]), + start_time=now, + end_time=now, + ) + + assert logger.Langfuse.trace.call_args.kwargs["id"] == "session-7125" + assert result["trace_id"] == "session-7125" + + +def test_session_header_classifier_survives_non_string_header_keys(): + """A non-string header key must not cost the caller its whole trace.""" + from litellm.integrations.langfuse.langfuse import _is_session_header_trace + + session: Final = "session-7125-abcdef" + headers: Final = {7: "numeric key", "x-litellm-session-id": session} + assert _is_session_header_trace(session, session, {"headers": headers}) is True + assert _is_session_header_trace(session, session, {"headers": {7: "numeric key"}}) is False + + def test_mask_input_header_false_keeps_the_prompt(): logger = _steering_logger() From 21e6c6e2f3c9fe993925b6aaaca6314bd56b1add Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:56:41 -0700 Subject: [PATCH 66/81] test(convert_dict_to_response): expect the type-naming error for a null choices value --- .../test_convert_dict_to_chat_completion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 09d1adba17c..31c554985a7 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1645,7 +1645,7 @@ class TestMissingChoicesGuard: assert result.usage.prompt_tokens == 10 def test_convert_to_model_response_object_null_choices_raises_api_error(self): - """choices=None raises APIError.""" + """choices=None raises APIError that names the type instead of claiming the key is missing.""" from litellm.exceptions import APIError response_object = { @@ -1661,7 +1661,7 @@ class TestMissingChoicesGuard: model_response_object=ModelResponse(), ) - assert "no 'choices'" in exc_info.value.message + assert "'choices' that is not a list (NoneType)" in exc_info.value.message def test_convert_to_streaming_response_no_choices_raises_api_error(self): """Missing choices in streaming cache-hit path raises APIError.""" From fbc6fb56ae2d8d8e600b0e80f7a46067b796d47f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:06:19 -0700 Subject: [PATCH 67/81] fix(bedrock): route gpt-6-astra reasoning_effort to reasoning.effort and mark Nova 2 tool_choice The converse reasoning gate only matched openai.gpt-5, so gpt-6-astra fell through to Anthropic's thinking block and Bedrock rejected the call with 400 Unknown parameter: 'thinking'. Match any openai.gpt- model at the three gate sites instead. Nova 2 lite and pro accept forced tool_choice on Converse (verified live on us.amazon.nova-2-lite-v1:0), so the nine Nova 2 registry keys now advertise supports_tool_choice. The invoke dispatcher also forwards json_mode to Nova like it already does for Anthropic and TwelveLabs. --- .../bedrock/chat/converse_transformation.py | 17 +++++-- .../base_invoke_transformation.py | 1 + ...odel_prices_and_context_window_backup.json | 9 ++++ model_prices_and_context_window.json | 9 ++++ .../test_base_invoke_transformation.py | 44 +++++++++++++++++++ .../chat/test_converse_transformation.py | 3 ++ 6 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6173f08fc87..4cdf387a11d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -4,6 +4,7 @@ Translating between OpenAI's `/chat/completion` format and Amazon's `/converse` import copy import json +import re import time import types from collections.abc import Mapping @@ -293,6 +294,10 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock", ) + @staticmethod + def _is_openai_gpt_reasoning_model(model: str) -> bool: + return re.search(r"openai\.gpt-\d", model) is not None + def _is_nova_2_model(self, model: str) -> bool: """ Check if the model is a Nova 2 model that supports reasoningConfig. @@ -423,14 +428,14 @@ class AmazonConverseConfig(BaseConfig): Handle the reasoning_effort parameter based on the model type. - GPT-OSS models: passed through unchanged via additionalModelRequestFields. - - OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields. + - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ if "gpt-oss" in model: optional_params["reasoning_effort"] = reasoning_effort - elif "openai.gpt-5" in model: + elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} optional_params["reasoning"] = reasoning elif self._is_nova_2_model(model): @@ -564,7 +569,11 @@ class AmazonConverseConfig(BaseConfig): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") - if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model: + if ( + "gpt-oss" in model + or self._is_openai_gpt_reasoning_model(model) + or self._is_openai_gpt_reasoning_model(base_model) + ): supported_params.append("reasoning_effort") elif self._is_nova_2_model(model): # Nova 2 models support reasoning_effort (transformed to reasoningConfig) @@ -920,7 +929,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["_parallel_tool_use_config"] = { "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } - if param == "thinking" and "openai.gpt-5" not in model: + if param == "thinking" and not self._is_openai_gpt_reasoning_model(model): if ( isinstance(value, dict) and value.get("type") == "adaptive" diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index a0e32c8aa22..90a2692f68a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -340,6 +340,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): optional_params=optional_params, litellm_params=litellm_params, encoding=encoding, + json_mode=json_mode, ) elif provider == "twelvelabs": return litellm.AmazonTwelveLabsPegasusConfig().transform_response( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a2e7b692649..0d2eda93323 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -381,6 +381,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -400,6 +401,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -417,6 +419,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -436,6 +439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -453,6 +457,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -472,6 +477,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -489,6 +495,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -508,6 +515,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -28578,6 +28586,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a2e7b692649..0d2eda93323 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -381,6 +381,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -400,6 +401,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -417,6 +419,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -436,6 +439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -453,6 +457,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -472,6 +477,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -489,6 +495,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -508,6 +515,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -28578,6 +28586,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index c2c448cd7e2..96a2fa6ec67 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,5 +1,7 @@ import json +from unittest.mock import MagicMock +import httpx import pytest @@ -190,3 +192,45 @@ def test_get_error_class_preserves_provider_headers(): assert isinstance(error, BedrockError) assert error.headers == {"x-amzn-RequestId": "req-invoke-500"} assert error.response.headers["x-amzn-requestid"] == "req-invoke-500" + + +def test_transform_response_hands_json_mode_to_nova(): + """The invoke dispatcher forwards its json_mode argument to Nova instead of dropping it.""" + from litellm.types.utils import ModelResponse + + response_json = { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_nova_json", + "name": "json_tool_call", + "input": {"city": "Paris", "temperature": 21}, + } + } + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 5, "outputTokens": 4, "totalTokens": 9}, + } + raw_response = httpx.Response(200, json=response_json, request=httpx.Request("POST", "https://bedrock")) + + result = AmazonInvokeConfig().transform_response( + model="invoke/amazon.nova-lite-v1:0", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "weather"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=True, + ) + + assert result.choices[0].message.tool_calls is None + assert json.loads(result.choices[0].message.content) == {"city": "Paris", "temperature": 21} diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 16824d19f23..51c0cd261e8 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -382,6 +382,8 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto(): "us.openai.gpt-5.6-sol", "global.openai.gpt-5.6-terra", "bedrock/converse/us.openai.gpt-5.6-luna", + "us.openai.gpt-6-astra", + "bedrock/converse/global.openai.gpt-6-astra", ], ) def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(model, local_model_cost_map): @@ -412,6 +414,7 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode [ "us.openai.gpt-5.6-sol", "bedrock/converse/global.openai.gpt-5.6-luna", + "us.openai.gpt-6-astra", ], ) def test_openai_gpt5_converse_never_forwards_thinking(model, local_model_cost_map): From 96e46ba8ff43cdbf24d53691685c15b533bfa746 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:06:55 -0700 Subject: [PATCH 68/81] fix(responses): stamp streamed usage cost when the provider usage arrives as a dict Perplexity's Responses payload fails ResponsesAPIResponse validation on truncation "" and is kept as an unvalidated model, so its usage stays a plain dict and _stamp_responses_usage_cost raised AttributeError on every streamed completion once reasoning made the cost non-zero. Validate the dict into ResponseAPIUsage before stamping, keeping a provider-reported cost when it carries one. Resolves LIT-7391 --- litellm/responses/streaming_iterator.py | 17 ++++++- .../responses/test_streaming_iterator.py | 51 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 9f9016c5a7f..658a0e8ad5d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload, runti import httpx from openai._streaming import SSEDecoder +from pydantic import ValidationError from typing_extensions import TypeIs import litellm @@ -544,7 +545,7 @@ class BaseResponsesAPIStreamingIterator: def _record_failed_response_usage(self, response_obj: ResponsesAPIResponse | None) -> None: if response_obj is None or self.logging_obj is None: return - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return try: @@ -1293,14 +1294,26 @@ def _add_text_like_part_events( ) +def _usage_as_model(usage: object) -> ResponseAPIUsage | None: + if isinstance(usage, ResponseAPIUsage): + return usage + if not isinstance(usage, dict): + return None + try: + return ResponseAPIUsage.model_validate(usage) + except ValidationError: + return None + + def _stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: if response_obj is None or logging_obj is None: return - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return + response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives if isinstance(getattr(usage_obj, "cost", None), (int, float)): return try: diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index c226c0b4d09..60ce08666b6 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -368,6 +368,57 @@ def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): logging_obj._response_cost_calculator.assert_not_called() +def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: + return ResponsesAPIResponse.model_construct( + id="resp_lit7391", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="perplexity/deepseek-v4-flash-0731", + object="response", + output=[], + truncation="", + usage=usage, + ) + + +def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.types.llms.openai import ResponseAPIUsage + + response = _unvalidated_response_with_dict_usage( + { + "input_tokens": 29, + "output_tokens": 120, + "output_tokens_details": {"reasoning_tokens": 117}, + "total_tokens": 149, + "cost": {"currency": "USD", "input_cost": 0, "output_cost": 3e-05, "total_cost": 3e-05}, + } + ) + logging_obj = Mock(spec=LiteLLMLoggingObj) + + _stamp_responses_usage_cost(response, logging_obj) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.cost == pytest.approx(3e-05) + assert response.usage.output_tokens_details.reasoning_tokens == 117 + logging_obj._response_cost_calculator.assert_not_called() + + +def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.types.llms.openai import ResponseAPIUsage + + response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.return_value = 0.000704 + + _stamp_responses_usage_cost(response, logging_obj) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.cost == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_called_once_with(result=response) + + def test_stamp_responses_usage_cost_survives_calculator_failure(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost From a248ff2c6ea712b0f8cb9f03bc327701c8dd63ab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:14:50 -0700 Subject: [PATCH 69/81] fix(guardrails): fail open on Responses tool-call rewrites in another shape and keep item names Guardrails that hand back tool_calls in their own shape (vendor JSON, user code output) raised a KeyError on the non-stream Responses write-back. Returned tool calls are now validated before comparison; a shape or count that does not line up leaves every tool-call item unchanged and logs a warning naming the guardrail. A tool call's name is written back only when the guardrail changed it, so a nameless custom_tool_call no longer picks up the custom_tool placeholder. --- .../guardrail_translation/handler.py | 85 ++++++++++++++---- ...test_openai_responses_guardrail_handler.py | 90 +++++++++++++++++++ 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index fd0d7b66bee..8c8294ad2fc 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -37,7 +37,7 @@ from itertools import accumulate, chain, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -105,6 +105,19 @@ class _ToolCallShape(NamedTuple): arguments: str +class _ToolCallFunctionFields(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str | None = None + arguments: str = "" + + +class _ToolCallFields(BaseModel): + model_config = ConfigDict(frozen=True) + + function: _ToolCallFunctionFields + + def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tuple[_ToolCallShape, ...]: return tuple( _ToolCallShape(name=tool_call["function"].get("name"), arguments=tool_call["function"].get("arguments", "")) @@ -112,6 +125,47 @@ def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tupl ) +def _returned_tool_call_shape(tool_call: object) -> _ToolCallShape | None: + payload: Final = tool_call.model_dump() if isinstance(tool_call, BaseModel) else tool_call + try: + fields: Final = _ToolCallFields.model_validate(payload) + except ValidationError: + return None + return _ToolCallShape(name=fields.function.name, arguments=fields.function.arguments) + + +def _post_guardrail_tool_call_shapes( + returned_tool_calls: Sequence[object] | None, + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str | None, +) -> tuple[_ToolCallShape, ...]: + if not pre_guardrail_tool_calls: + return pre_guardrail_tool_calls + if returned_tool_calls is None or len(returned_tool_calls) != len(pre_guardrail_tool_calls): + verbose_proxy_logger.warning( + "OpenAI Responses API: guardrail %s returned %s tool calls for the %d scanned, " + "leaving the tool call output items unchanged", + guardrail_name, + "no" if returned_tool_calls is None else len(returned_tool_calls), + len(pre_guardrail_tool_calls), + ) + return pre_guardrail_tool_calls + returned_shapes: Final = tuple(_returned_tool_call_shape(tool_call) for tool_call in returned_tool_calls) + validated_shapes: Final = tuple(shape for shape in returned_shapes if shape is not None) + if len(validated_shapes) != len(returned_shapes): + verbose_proxy_logger.warning( + "OpenAI Responses API: guardrail %s returned tool calls without a function name and arguments, " + "leaving the tool call output items unchanged", + guardrail_name, + ) + return pre_guardrail_tool_calls + return validated_shapes + + +def _tool_call_rewrite(before: _ToolCallShape, after: _ToolCallShape) -> _ToolCallShape: + return _ToolCallShape(name=after.name if after.name != before.name else None, arguments=after.arguments) + + class ResponseOutputEnvelope(TypedDict, total=False): """Dict form of a Responses API response, as far as guardrail write-back reads it.""" @@ -698,7 +752,11 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") + post_guardrail_tool_calls: Final = _post_guardrail_tool_call_shapes( + returned_tool_calls=guardrailed_inputs.get("tool_calls"), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name, + ) # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( @@ -709,11 +767,7 @@ class OpenAIResponsesHandler(BaseTranslation): self._write_tool_call_rewrites_to_output( tool_call_items=tuple(item for item in response_output if _is_tool_call_output_item(item)), pre_guardrail_tool_calls=pre_guardrail_tool_calls, - post_guardrail_tool_calls=_tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check - ), + post_guardrail_tool_calls=post_guardrail_tool_calls, ) verbose_proxy_logger.debug("OpenAI Responses API: Processed output response: %s", response) @@ -811,11 +865,10 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") - post_guardrail_tool_calls: Final = _tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + post_guardrail_tool_calls: Final = _post_guardrail_tool_call_shapes( + returned_tool_calls=guardrailed_inputs.get("tool_calls"), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name, ) # Write guardrailed texts back into the output items in-place. @@ -991,7 +1044,7 @@ class OpenAIResponsesHandler(BaseTranslation): ) rewrites_by_call_id: Final = MappingProxyType( { - call_id: after + call_id: _tool_call_rewrite(before, after) for call_id, before, after in zip(call_ids, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before } @@ -1050,12 +1103,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) -> None: if len(tool_call_items) != len(post_guardrail_tool_calls): return - for output_item, after in ( - (output_item, after) + for output_item, rewrite in ( + (output_item, _tool_call_rewrite(before, after)) for output_item, before, after in zip(tool_call_items, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before ): - self._write_tool_call_item(output_item, after.name, after.arguments) + self._write_tool_call_item(output_item, rewrite.name, rewrite.arguments) @staticmethod def _tool_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index cf4895f637f..a4f0a77a9b6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -10,6 +10,8 @@ from collections.abc import Callable from typing import Any, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock +import logging + import pytest @@ -85,6 +87,29 @@ class PersimmonMaskingGuardrail(CustomGuardrail): return {**inputs, "tool_calls": tool_calls} +class FlatShapeGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + flat_tool_calls = [{"name": "exec", "input": "rm -rf /"} for _ in inputs.get("tool_calls", [])] + return {**inputs, "tool_calls": flat_tool_calls} + + +class DroppingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "tool_calls": []} + + CUSTOM_TOOL_CALL_ITEM = { "type": "custom_tool_call", "id": "ctc_1", @@ -737,6 +762,52 @@ class TestOpenAIResponsesHandlerToolCallExtraction: assert (custom_item.name if typed else custom_item["name"]) == "exec" assert (output[0].content[0].text if typed else output[0]["content"][0]["text"]) == "running persimmon" + @staticmethod + def _custom_tool_call_response(item: dict) -> dict: + return { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [item], + } + + @pytest.mark.asyncio + async def test_process_output_response_ignores_tool_call_rewrites_in_another_shape(self): + handler = OpenAIResponsesHandler() + response = self._custom_tool_call_response(dict(CUSTOM_TOOL_CALL_ITEM)) + + result = await handler.process_output_response(response, FlatShapeGuardrail(guardrail_name="flat")) + + assert result["output"][0]["input"] == "echo persimmon" + assert result["output"][0]["name"] == "exec" + + @pytest.mark.asyncio + async def test_process_output_response_warns_when_guardrail_drops_tool_calls(self, caplog): + handler = OpenAIResponsesHandler() + response = self._custom_tool_call_response(dict(CUSTOM_TOOL_CALL_ITEM)) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await handler.process_output_response(response, DroppingGuardrail(guardrail_name="dropper")) + + assert result["output"][0]["input"] == "echo persimmon" + assert any( + "dropper" in record.getMessage() and "0 tool calls for the 1 scanned" in record.getMessage() + for record in caplog.records + ) + + @pytest.mark.asyncio + async def test_process_output_response_keeps_a_nameless_custom_tool_call_nameless(self): + handler = OpenAIResponsesHandler() + nameless_item = {key: value for key, value in CUSTOM_TOOL_CALL_ITEM.items() if key != "name"} + response = self._custom_tool_call_response(nameless_item) + + result = await handler.process_output_response(response, PersimmonMaskingGuardrail(guardrail_name="mask")) + + assert result["output"][0]["input"] == "echo [MASKED]" + assert "name" not in result["output"][0] + @pytest.mark.asyncio async def test_process_output_response_with_tool_calls(self): """Test processing output response containing function tool calls""" @@ -1469,6 +1540,25 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[5]["response"]["output"][0]["name"] == "exec" assert "arguments" not in events[5]["response"]["output"][0] + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_keep_a_nameless_custom_tool_call_nameless(self): + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + items = [events[0]["item"], events[4]["item"], events[5]["response"]["output"][0]] + for item in items: + del item["name"] + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert events[3]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["input"] == "echo [MASKED]" + assert all("name" not in item for item in items) + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_syncs_typed_custom_tool_call_events(self): from litellm.types.llms.openai import ( From 7563d94e65591c2ed3b909c196a3eb76e61625fd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:29:55 -0700 Subject: [PATCH 70/81] test(responses): assert the streamed response.completed event echoes the named tool_choice --- .../test_streaming_iterator_transformation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 29061db97dc..70f0957890d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -679,6 +679,9 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: "response.in_progress", "response.completed", ] - assert response_events[0].response.tool_choice == {"type": "function", "name": "run_command"} - assert response_events[1].response.tool_choice == {"type": "function", "name": "run_command"} + assert [event.response.tool_choice for event in response_events] == [ + {"type": "function", "name": "run_command"}, + {"type": "function", "name": "run_command"}, + {"type": "function", "name": "run_command"}, + ] assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) From a68fe4e4d9eb8b55efd33be65ef4e1f1f1932c62 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:31:48 -0700 Subject: [PATCH 71/81] fix(responses): keep the client's usage shape when the logging copy cannot re-validate the response --- litellm/responses/streaming_iterator.py | 41 ++++++++------ .../responses/test_streaming_iterator.py | 54 ++++++++++++++++--- 2 files changed, 72 insertions(+), 23 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 658a0e8ad5d..40ff88fc557 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload, runti import httpx from openai._streaming import SSEDecoder -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from typing_extensions import TypeIs import litellm @@ -439,18 +439,7 @@ class BaseResponsesAPIStreamingIterator: if self._persist_completed_response_before_logging: self._persist_completed_response_to_cache(is_async=is_async) - # Create a copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) - # to chat completion format (prompt_tokens/completion_tokens) for internal logging - # Use model_dump + model_validate instead of deepcopy to avoid pickle errors with - # Pydantic ValidatorIterator when response contains tool_choice with allowed_tools (fixes #17192) - logging_response = self.completed_response - if self.completed_response is not None and hasattr(self.completed_response, "model_dump"): - try: - logging_response = type(self.completed_response).model_validate(self.completed_response.model_dump()) - except Exception: - # Fallback to original if serialization fails - pass + logging_response: Final[object] = _logging_copy(self.completed_response) self._restore_provider_response_headers(logging_response) end_time: Final = datetime.now() @@ -489,10 +478,10 @@ class BaseResponsesAPIStreamingIterator: def _restore_provider_response_headers(self, logging_response: object) -> None: """Re-apply the provider's response headers to the copy handed to logging callbacks. - ``model_validate(model_dump())`` above drops pydantic private attributes, so the + ``model_validate(model_dump())`` in ``_logging_copy`` drops pydantic private attributes, so the ``_hidden_params`` the provider transform set on the nested response are lost. Returns early - when that copy fell back to the original event, so logging-only state never lands on the - object the caller is iterating. + when the event was not a pydantic model and logging got the original, so logging-only state + never lands on the object the caller is iterating. """ if logging_response is self.completed_response: return @@ -1294,6 +1283,26 @@ def _add_text_like_part_events( ) +def _logging_copy(event: object) -> object: + """Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never + reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the + deepcopy pickle errors of #17192; when a provider payload fails validation (LIT-7391), shallow + copies of the event and its nested response still keep the caller's ``usage`` attribute separate.""" + if not isinstance(event, BaseModel): + return event + try: + return type(event).model_validate(event.model_dump()) + except Exception: + return _detached_shallow_copy(event) + + +def _detached_shallow_copy(event: BaseModel) -> BaseModel: + nested: Final[object] = getattr(event, "response", None) + if isinstance(nested, BaseModel): + return event.model_copy(update={"response": nested.model_copy()}) + return event.model_copy() + + def _usage_as_model(usage: object) -> ResponseAPIUsage | None: if isinstance(usage, ResponseAPIUsage): return usage diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 60ce08666b6..5e0e794d93e 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -18,6 +18,7 @@ from litellm.responses.streaming_iterator import ( SyncResponsesAPIStreamingIterator, ) from litellm.types.llms.openai import ( + ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -329,8 +330,6 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): def _responses_api_response_with_usage() -> ResponsesAPIResponse: - from litellm.types.llms.openai import ResponseAPIUsage - return ResponsesAPIResponse( id="resp_lit6427", created_at=int(datetime(2025, 1, 1).timestamp()), @@ -383,8 +382,6 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost - from litellm.types.llms.openai import ResponseAPIUsage - response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -406,8 +403,6 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost - from litellm.types.llms.openai import ResponseAPIUsage - response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) logging_obj = Mock(spec=LiteLLMLoggingObj) logging_obj._response_cost_calculator.return_value = 0.000704 @@ -586,5 +581,50 @@ async def test_streaming_logging_copy_fallback_leaves_caller_event_untouched(): with patch.object(type(iterator.completed_response), "model_dump", side_effect=ValueError("cannot serialize")): iterator._log_completed_response(is_async=True) - assert logged == [iterator.completed_response] + assert len(logged) == 1 + assert logged[0] is not iterator.completed_response + assert logged[0].response is not iterator.completed_response.response + assert logged[0].response._hidden_params["headers"]["apim-request-id"] == "azure-correlation-1" assert iterator.completed_response.response._hidden_params == {} + + +def _unvalidated_completed_config() -> Mock: + """Config whose completed event carries a Perplexity-style response that fails validation + (``truncation: ""``) and already holds the stamped ``ResponseAPIUsage``.""" + mock_config = Mock(spec=BaseResponsesAPIConfig) + + def _transform(model, parsed_chunk, logging_obj): + response = _unvalidated_response_with_dict_usage( + ResponseAPIUsage(input_tokens=29, output_tokens=373, total_tokens=402, cost={"total_cost": 0.0001}) + ) + return ResponseCompletedEvent(type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=response) + + mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_keeps_client_usage_when_response_fails_validation(): + """LIT-7391: the logging copy cannot round-trip a response that fails validation, and logging + rewrites the assembled response's usage to chat shape in place, so the event handed to logging + must never be the one the caller receives.""" + logging_obj = _logging_obj_stub() + logging_obj.stream = True + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator(headers={}, config=_unvalidated_completed_config(), logging_obj=logging_obj) + events = [event async for event in iterator] + + assert len(logged) == 1 + now = datetime.now() + LiteLLMLoggingObj._get_assembled_streaming_response( + logging_obj, logged[0], start_time=now, end_time=now, is_async=True, streaming_chunks=[] + ) + assert logged[0].response.usage["prompt_tokens"] == 29 + + client_usage = events[-1].response.usage + assert isinstance(client_usage, ResponseAPIUsage) + assert client_usage.input_tokens == 29 + assert client_usage.cost == pytest.approx(0.0001) From 969d152c4bdc781e706ffffcd496ab1f3332c444 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 16:40:25 -0700 Subject: [PATCH 72/81] fix(jwt): evict jwt_key_mapping cache when a virtual key is deleted The FK cascade drops the LiteLLM_JWTKeyMapping row, but the cached jwt_key_mapping:{claim}:{value} entry still resolved to the deleted token hash, so every JWT call from that identity failed until virtual_key_mapping_cache_ttl expired instead of auto-registering against a recreated key. delete_verification_tokens now snapshots the mapping cache keys before the delete and evicts them across replicas afterwards, the same way /key/regenerate already does. Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- .../key_management_endpoints.py | 19 ++++ .../test_key_management_endpoints.py | 98 +++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index f46c4170071..749a940de0e 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4559,6 +4559,23 @@ async def delete_verification_tokens( litellm_changed_by=litellm_changed_by, ) + # Snapshot before the delete: the FK cascade drops the mapping rows, but their + # cached jwt_key_mapping entries still resolve to the now-dead token (LIT-5380). + jwt_mapping_cache_keys: Final[tuple[str, ...]] = tuple( + cache_key + for keys_for_token in await asyncio.gather( + *( + get_jwt_key_mapping_cache_keys_for_token( + hashed_token=key.token, + prisma_client=prisma_client, + ) + for key in authorized_keys + if key.token is not None + ) + ) + for cache_key in keys_for_token + ) + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: deleted_tokens = await prisma_client.delete_data(tokens=tokens) if deleted_tokens is not None and len(deleted_tokens) != len(tokens): @@ -4571,6 +4588,8 @@ async def delete_verification_tokens( if len(deleted_tokens) != len(tokens): failed_tokens = [token for token in tokens if token not in deleted_tokens] + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) + else: raise Exception("DB not connected. prisma_client is None") except Exception as e: diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d2aeba18f7d..65cc23ea67f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5085,6 +5085,104 @@ async def test_delete_verification_tokens_persists_deleted_keys(monkeypatch): assert len(deleted_keys) == 2 +class _JWTMappingRow: + def __init__(self, token, jwt_claim_name, jwt_claim_value): + self.token = token + self.jwt_claim_name = jwt_claim_name + self.jwt_claim_value = jwt_claim_value + + +class _CascadingJWTMappingTable: + """Mapping rows that LiteLLM_JWTKeyMapping_token_fkey drops when their key is deleted.""" + + def __init__(self, rows): + self.rows = rows + + async def find_many(self, where, **kwargs): + return [row for row in self.rows if row.token == where["token"]] + + def cascade(self, deleted_tokens): + self.rows = [row for row in self.rows if row.token not in deleted_tokens] + + +class _RecordingEvict: + def __init__(self): + self.cache_keys = () + + async def __call__(self, cache_keys, user_api_key_cache): + self.cache_keys = tuple(cache_keys) + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypatch): + """Deleting a key must evict its jwt_key_mapping cache entries (LIT-5380). + + The FK cascade removes the mapping rows, so a surviving cache entry would keep + resolving the deleted token hash and 401 every JWT call from that identity until + virtual_key_mapping_cache_ttl expires, instead of auto-registering again. + """ + jwt_table = _CascadingJWTMappingTable( + [_JWTMappingRow("hashed-token-1", "email", "user@example.com")] + ) + + key1 = LiteLLM_VerificationToken( + token="hashed-token-1", + user_id="user-123", + team_id=None, + key_alias="jwt-mapped-key", + spend=0.0, + max_budget=None, + models=[], + aliases={}, + config={}, + permissions={}, + metadata={}, + model_max_budget={}, + model_spend={}, + soft_budget_cooldown=False, + allowed_routes=[], + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[key1] + ) + mock_prisma_client.db.litellm_jwtkeymapping = jwt_table + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + + async def cascading_delete_data(tokens): + jwt_table.cascade(tokens) + return list(tokens) + + mock_prisma_client.delete_data = AsyncMock(side_effect=cascading_delete_data) + + recording_evict = _RecordingEvict() + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints.evict_and_broadcast", + recording_evict, + ) + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + lambda token: token, + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ) + + await delete_verification_tokens( + tokens=["hashed-token-1"], + user_api_key_cache=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + ) + + assert recording_evict.cache_keys == ("jwt_key_mapping:email:user@example.com",) + + @pytest.mark.asyncio async def test_delete_key_fn_persists_deleted_keys(monkeypatch): from litellm.proxy._types import KeyRequest From 2000642592670baf38c46f4c6e3bcb851c59d79e Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 16:43:50 -0700 Subject: [PATCH 73/81] fix(router): resolve route candidate ids through the router's own resolver (#40491) get_candidate_model_ids_for_route (added in #40280 for the encrypted-content affinity check) reconstructed the candidate pool by unioning the model_name and team indexes with pattern_router.route. That diverged from how the router actually resolves a route: it took a union instead of the first matching path, and pattern_router.route only matches the literal name, so a provider-qualified pattern (matched by get_deployments_by_pattern, which retries the {provider}/{model} form) was missed and the default deployment was ignored. For an affinity follow-up on a wildcard or team-public route, that mismatch could strip encrypted reasoning on a same-group cooldown, or return a 503 on a real cross-path switch. Delegate the non-model_name case to _try_early_resolve_deployments_for_model_not_in_names, the same resolver _common_checks_available_deployment uses, so candidate membership follows the router's real precedence. With include_team_models left off it stays read-only and does not raise. Behavior for concrete model groups and routing groups is unchanged. Claude-Session: https://claude.ai/code/session_01KAumQbhzk6jdWWHFLA8Jar Co-authored-by: Claude Opus 4.8 --- litellm/router.py | 35 +++++++++++++++++-------------- tests/test_litellm/test_router.py | 7 ++++++- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 88601e5b97b..f9d4bf1428e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11170,28 +11170,31 @@ class Router: def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: """ - Deployment ids that could serve ``model`` for ``team_id``, unioned across the paths - the router resolves a route through: ``model_group_alias``, a routing group, the - ``model_name`` and team indexes, and wildcard pattern routes. Read-only and - side-effect-free, unlike ``_common_checks_available_deployment`` which also applies - fallbacks and can raise. Lets a pre-call check tell a genuine cross-group route from - same-group unavailability without re-deriving that precedence at the call site, and - without leaking deployment ids into request kwargs bound for the provider. + Deployment ids that could serve ``model`` for ``team_id``, following the same + precedence ``_common_checks_available_deployment`` uses to build a candidate pool: + ``model_group_alias``, then a routing group, then the first matching early-resolve + path for a name that is not a ``model_name`` (team route, wildcard pattern via + ``get_deployments_by_pattern``, team pattern router, default deployment), then the + ``model_name`` and team indexes. Delegating to the router's own resolvers keeps this + aligned with how a route actually resolves rather than re-deriving it, and unlike + ``_common_checks_available_deployment`` it is read-only: it does not apply request + fallbacks and (with ``include_team_models`` left off) does not raise. Lets a pre-call + check tell a genuine cross-group route from same-group unavailability without leaking + deployment ids into request kwargs bound for the provider. """ resolved: Final = self._get_model_from_alias(model=model) or model routing_group_members: Final = self._get_routing_group_deployments(model=resolved, team_id=team_id) if routing_group_members is not None: return self._deployment_ids(routing_group_members) - if resolved in self.model_names: - return self._deployment_ids(self._get_all_deployments(model_name=resolved, team_id=team_id)) - team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - return self._deployment_ids( - ( - *self._get_all_deployments(model_name=resolved, team_id=team_id), - *(self.pattern_router.route(resolved) or ()), - *((team_router.route(resolved) or ()) if team_router is not None else ()), - ) + early: Final = self._try_early_resolve_deployments_for_model_not_in_names( + model=resolved, request_team_id=team_id ) + if early is not None: + early_deployments: Final = early[1] + return self._deployment_ids( + (early_deployments,) if isinstance(early_deployments, Mapping) else early_deployments + ) + return self._deployment_ids(self._get_all_deployments(model_name=resolved, team_id=team_id)) @staticmethod def _deployment_ids(deployments: Sequence[Mapping[str, object]]) -> frozenset[str]: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1e0327e5ac2..f5b1f79ad73 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15023,7 +15023,9 @@ def test_get_candidate_model_ids_for_route_covers_model_name_and_pattern(): pre-call check can tell a genuine cross-group route from same-group unavailability. A concrete model group returns its member ids; a wildcard/pattern deployment is included for a concrete model it matches, which the bare model_name index misses. - Regression guard for the LIT-7195 tier-change discriminator's team/pattern gaps. + The unprefixed-name case must resolve through get_deployments_by_pattern (which retries + the provider-qualified form), not a bare pattern_router.route that only sees the literal + name. Regression guard for the LIT-7195 tier-change discriminator's team/pattern gaps. """ router = Router( model_list=[ @@ -15047,6 +15049,9 @@ def test_get_candidate_model_ids_for_route_covers_model_name_and_pattern(): assert router.get_candidate_model_ids_for_route(model="grp") == frozenset({"dep-a", "dep-b"}) assert "dep-wild" in router.get_candidate_model_ids_for_route(model="openai/gpt-4o-some-new-model") + # unprefixed name whose provider resolves to openai: only get_deployments_by_pattern's + # provider-qualified retry matches "openai/*"; a bare route() on the literal name misses it + assert "dep-wild" in router.get_candidate_model_ids_for_route(model="gpt-5") def test_deployment_ids_stringifies_ids_and_skips_entries_without_a_model_info_id(): From 70c238dde0c4bef5cbb976fbd382320a8021352d Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:54:43 -0700 Subject: [PATCH 74/81] fix(ui): preserve dotted MCP tool argument names --- ...MCPToolArgumentsForm.integration.test.tsx} | 114 +++++++++++++++++- .../mcp_tools/MCPToolArgumentsForm.tsx | 44 ++++--- 2 files changed, 141 insertions(+), 17 deletions(-) rename ui/litellm-dashboard/src/components/mcp_tools/{MCPToolArgumentsForm.test.tsx => MCPToolArgumentsForm.integration.test.tsx} (61%) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx similarity index 61% rename from ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx rename to ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx index d6e614da5eb..a4302f9fd6a 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect } from "vitest"; import MCPToolArgumentsForm, { MCPToolArgumentsFormRef } from "./MCPToolArgumentsForm"; @@ -26,6 +26,118 @@ const submitError = async (ref: React.RefObject) }; describe("MCPToolArgumentsForm", () => { + it("keeps dotted arguments separate from a same-prefix object and converts their values", async () => { + const ref = renderForm({ + type: "object", + properties: { + "filter.category": { type: "string" }, + filter: { type: "object" }, + "page.limit": { type: "integer" }, + query: { type: "string" }, + }, + required: ["filter.category"], + }); + + fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { + target: { value: "invoices" }, + }); + fireEvent.change(screen.getByRole("textbox", { name: "filter", exact: true }), { + target: { value: '{"category":"receipts","metadata":{"region":"eu"}}' }, + }); + fireEvent.change(screen.getByRole("spinbutton", { name: "page.limit" }), { target: { value: "7" } }); + fireEvent.change(screen.getByRole("textbox", { name: "query" }), { target: { value: "September" } }); + + const expected = { + "filter.category": "invoices", + filter: { category: "receipts", metadata: { region: "eu" } }, + "page.limit": 7, + query: "September", + }; + await expect(submit(ref)).resolves.toEqual(expected); + }); + + it("shows required validation on the literal dotted field and accepts a correction", async () => { + const ref = renderForm({ + type: "object", + properties: { "filter.category": { type: "string" } }, + required: ["filter.category"], + }); + + expect(await submitError(ref)).toEqual({ + errorFields: [{ name: ["filter.category"], errors: ["Please enter filter.category"] }], + }); + expect(await screen.findByText("Please enter filter.category")).toBeInTheDocument(); + expect(screen.getByRole("textbox", { name: "filter.category *" })).toHaveAttribute("aria-invalid", "true"); + + fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { + target: { value: "invoices" }, + }); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "invoices" }); + }); + + it("validates JSON for dotted arguments inside params and preserves their literal names", async () => { + const ref = renderForm({ + type: "object", + properties: { + params: { + type: "object", + properties: { "filter.options": { type: "object" } }, + required: ["filter.options"], + }, + }, + required: [], + }); + const field = screen.getByRole("textbox", { name: "filter.options *" }); + fireEvent.change(field, { target: { value: "invalid" } }); + + expect(await submitError(ref)).toEqual({ + errorFields: [{ name: ["filter.options"], errors: ["Invalid JSON"] }], + }); + expect(await screen.findByText("Invalid JSON")).toBeInTheDocument(); + + fireEvent.change(field, { target: { value: '{"region":"eu"}' } }); + await expect(submit(ref)).resolves.toEqual({ params: { "filter.options": { region: "eu" } } }); + }); + + it("resets dotted defaults and positional values when the selected tool changes", async () => { + const ref = React.createRef(); + const { rerender } = render( + , + ); + expect(screen.getByRole("textbox", { name: "filter.category" })).toHaveValue("invoices"); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "invoices" }); + fireEvent.change(screen.getByRole("textbox", { name: "filter.category" }), { + target: { value: "edited" }, + }); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "edited" }); + + rerender( + , + ); + expect(screen.getByRole("textbox", { name: "filter.category" })).toHaveValue("receipts"); + await expect(submit(ref)).resolves.toEqual({ query: "new tool", "filter.category": "receipts" }); + }); + it("returns typed values for a string, integer, number and boolean field", async () => { const user = userEvent.setup(); const ref = renderForm({ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx index ca8c5697e6e..4fff3e57fd9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx @@ -9,7 +9,10 @@ import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { MCPTool, InputSchema, InputSchemaProperty } from "./types"; -type ToolFormValues = Record; +type ToolFormValues = { args: unknown[] }; + +const argumentValues = (schema: InputSchema, values: ToolFormValues): Record => + Object.fromEntries(Object.keys(schema.properties ?? {}).map((key, index) => [key, values.args[index]])); const STRING_SCHEMA_MESSAGES: Readonly> = { input: "Please enter input for this tool" }; @@ -38,7 +41,7 @@ type FieldError = { type: string; message: string }; const collectErrors = ( actualSchema: InputSchema, requiredMessages: Readonly>, - values: ToolFormValues, + values: Record, ): Record => { const entries = Object.entries(actualSchema.properties ?? {}).flatMap<[string, FieldError]>(([key, prop]) => { const value = values[key]; @@ -57,8 +60,18 @@ const collectErrors = ( const buildResolver = (actualSchema: InputSchema, requiredMessages: Readonly> = {}): Resolver => (values) => { - const errors = collectErrors(actualSchema, requiredMessages, values); - return Object.keys(errors).length > 0 ? { values: {}, errors } : { values, errors: {} }; + const errors = collectErrors(actualSchema, requiredMessages, argumentValues(actualSchema, values)); + if (Object.keys(errors).length === 0) return { values, errors: {} }; + return { + values: {}, + errors: { + args: Object.fromEntries( + Object.keys(actualSchema.properties ?? {}).flatMap((key, index) => + Object.hasOwn(errors, key) ? [[index, errors[key]]] : [], + ), + ), + }, + }; }; const labelFor = (key: string, prop: InputSchemaProperty, required: boolean): React.ReactNode => ( @@ -238,10 +251,7 @@ const MCPToolArgumentsForm = forwardRef( - () => - Object.fromEntries( - Object.entries(actualSchema.properties ?? {}).map(([key, prop]) => [key, getInitialValueForField(prop)]), - ), + () => ({ args: Object.values(actualSchema.properties ?? {}).map(getInitialValueForField) }), [actualSchema], ); @@ -255,7 +265,7 @@ const MCPToolArgumentsForm = forwardRef ({ getSubmitValues: async () => { - const values = form.getValues(); + const values = argumentValues(actualSchema, form.getValues()); const errors = collectErrors(actualSchema, requiredMessages, values); if (Object.keys(errors).length > 0) { await form.trigger(); @@ -286,14 +296,16 @@ const MCPToolArgumentsForm = forwardRef Input * } > - {(field) => } + {(field) => ( + + )} @@ -318,13 +330,13 @@ const MCPToolArgumentsForm = forwardRef - {Object.entries(actualSchema.properties).map(([key, prop]) => { + {Object.entries(actualSchema.properties).map(([key, prop], index) => { const required = actualSchema.required?.includes(key) ?? false; return ( {(field) => { @@ -375,7 +387,7 @@ const MCPToolArgumentsForm = forwardRef ); @@ -385,7 +397,7 @@ const MCPToolArgumentsForm = forwardRef ); From 0b21b99ebd193872c520996cec79defad07ee108 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:57:15 -0700 Subject: [PATCH 75/81] fix(ui): type MCP argument resolver results explicitly --- .../mcp_tools/MCPToolArgumentsForm.integration.test.tsx | 2 +- .../src/components/mcp_tools/MCPToolArgumentsForm.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx index a4302f9fd6a..b1174d1d37d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx @@ -41,7 +41,7 @@ describe("MCPToolArgumentsForm", () => { fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { target: { value: "invoices" }, }); - fireEvent.change(screen.getByRole("textbox", { name: "filter", exact: true }), { + fireEvent.change(screen.getByRole("textbox", { name: "filter" }), { target: { value: '{"category":"receipts","metadata":{"region":"eu"}}' }, }); fireEvent.change(screen.getByRole("spinbutton", { name: "page.limit" }), { target: { value: "7" } }); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx index 4fff3e57fd9..ab3213d9b37 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useImperativeHandle, useMemo } from "react"; import { CircleHelp } from "lucide-react"; -import { useForm, type Resolver } from "react-hook-form"; +import { useForm, type Resolver, type ResolverResult } from "react-hook-form"; import { FieldGroup } from "@/components/ui/field"; import { FormField } from "@/components/shared/form/FormField"; import { Input } from "@/components/ui/input"; @@ -59,7 +59,7 @@ const collectErrors = ( const buildResolver = (actualSchema: InputSchema, requiredMessages: Readonly> = {}): Resolver => - (values) => { + (values): ResolverResult => { const errors = collectErrors(actualSchema, requiredMessages, argumentValues(actualSchema, values)); if (Object.keys(errors).length === 0) return { values, errors: {} }; return { From 6b721de3e530bc97758d59621e57e3a75a9ebadf Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 17:15:48 -0700 Subject: [PATCH 76/81] fix(databricks): route Unity model services through AI Gateway (#40492) Co-authored-by: Claude Code --- .../llms/databricks/chat/transformation.py | 6 +- litellm/llms/databricks/common_utils.py | 20 ++++-- .../test_databricks_chat_transformation.py | 13 ++++ .../test_databricks_partner_integration.py | 71 +++++++++++++++++++ 4 files changed, 101 insertions(+), 9 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 54c7040daf9..09aaf970dc5 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -250,8 +250,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = self._get_api_base(api_base) - complete_url: Final = f"{api_base}/chat/completions" + use_ai_gateway: Final = model.removeprefix("databricks/").count(".") >= 2 + api_base = self._get_api_base(api_base, use_ai_gateway=use_ai_gateway) + url_base: Final = api_base.rstrip("/") if use_ai_gateway else api_base + complete_url: Final = f"{url_base}/chat/completions" return complete_url def get_supported_openai_params(self, model: str | None = None) -> list: diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 7695b1cb35e..a4ec2c5378b 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -177,19 +177,13 @@ class DatabricksBase: # Default: just litellm return f"litellm/{version}" - def _get_api_base(self, api_base: str | None) -> str: - """ - Get the Databricks API base URL. - - If not provided, attempts to get it from the Databricks SDK. - """ + def _get_api_base(self, api_base: str | None, use_ai_gateway: bool = False) -> str: if api_base is None: try: from databricks.sdk import WorkspaceClient databricks_client: Final = WorkspaceClient() api_base = f"{databricks_client.config.host}/serving-endpoints" - return api_base except ImportError: raise DatabricksException( status_code=400, @@ -198,6 +192,18 @@ class DatabricksBase: "or install the databricks-sdk Python library." ), ) + + if not use_ai_gateway: + return api_base + + normalized_api_base: Final = api_base.rstrip("/") + if normalized_api_base.endswith("/ai-gateway/mlflow/v1"): + return normalized_api_base + if normalized_api_base.endswith("/serving-endpoints"): + return f"{normalized_api_base.removesuffix('/serving-endpoints')}/ai-gateway/mlflow/v1" + api_base_parts: Final = urlsplit(normalized_api_base) + if api_base_parts.path in ("", "/"): + return f"{normalized_api_base}/ai-gateway/mlflow/v1" return api_base def _get_oauth_m2m_token( diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 38d5844b9b9..caf7bed7385 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -255,6 +255,19 @@ def test_transform_messages_sanitizes_empty_content(): assert result[1]["content"] == "Hi" +def test_transform_request_preserves_unity_model_service_name(): + config = DatabricksConfig() + result = config.transform_request( + model="system.ai.kimi-k3", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["model"] == "system.ai.kimi-k3" + + def test_transform_request_strips_thinking_blocks_and_reasoning_content(): """Regression for LIT-6762: replaying an assistant turn that litellm decorated with `thinking_blocks` / `reasoning_content` made Databricks 400 with diff --git a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py index 39198bb20f3..c6ad78366f7 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py +++ b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py @@ -657,6 +657,77 @@ class TestEndpointURLConstruction: assert api_base.endswith("/chat/completions") + def test_chat_gateway_endpoint_for_unity_model_on_legacy_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/serving-endpoints", + api_key="test-key", + model="system.ai.kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + + def test_chat_gateway_endpoint_preserves_explicit_gateway_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/ai-gateway/mlflow/v1/", + api_key="test-key", + model="system.ai.kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + + def test_chat_gateway_preserves_unity_model_service_name_with_explicit_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + config = DatabricksConfig() + request = config.transform_request( + model="catalog.schema.kimi-k3", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert config.get_complete_url( + api_base="https://test.net/ai-gateway/mlflow/v1", + api_key="test-key", + model="catalog.schema.kimi-k3", + optional_params={}, + litellm_params={}, + ) == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + assert request["model"] == "catalog.schema.kimi-k3" + + def test_chat_legacy_endpoint_remains_default(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/serving-endpoints", + api_key="test-key", + model="databricks-kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/serving-endpoints/chat/completions" + def test_embeddings_endpoint(self, monkeypatch): """Embeddings endpoint is correctly appended.""" monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) From b8d7f68aeba2bd4a599895c910df085c2eb530a1 Mon Sep 17 00:00:00 2001 From: kerry-berri Date: Wed, 9 Sep 2026 17:46:12 -0700 Subject: [PATCH 77/81] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../src/components/llm_calls/chat_completion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index b813a35f687..c12a2c3cfb4 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -245,7 +245,7 @@ export async function makeOpenAIChatCompletionRequest( // Extract cost from usage object if available if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - const parsedCost = parseFloat(chunkWithUsage.usage.cost); + const parsedCost = Number(chunkWithUsage.usage.cost); if (Number.isFinite(parsedCost)) { usageData.cost = parsedCost; } From 317b29e69d658ca2502e656d4a30c4db5ced0198 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 18:02:23 -0700 Subject: [PATCH 78/81] feat(cli): add lite configure claude and lite unconfigure claude (#40319) Persistently route Claude Code through a LiteLLM proxy with a long-lived virtual key or the stored lite login, turn on gateway model discovery so /model lists the proxy's models, optionally pick the model Claude Code starts on, and record what changed so unconfigure restores only the keys the user has not touched since. lite login --config-claude writes through the same receipt and is undoable too. The two settings merges (lite up / --config-claude and lite autoroute) collapse into one credential-aware merge --- litellm/proxy/client/cli/README.md | 22 +- litellm/proxy/client/cli/commands/agents.py | 5 +- litellm/proxy/client/cli/commands/auth.py | 29 +- .../client/cli/commands/autoroute/commands.py | 9 +- .../client/cli/commands/autoroute/settings.py | 51 -- .../client/cli/commands/claude_settings.py | 517 +++++++++++++++-- .../proxy/client/cli/commands/configure.py | 252 +++++++++ litellm/proxy/client/cli/commands/pi.py | 31 +- litellm/proxy/client/cli/commands/up.py | 9 +- litellm/proxy/client/cli/main.py | 4 + .../client/cli/autoroute/test_commands.py | 4 + .../client/cli/autoroute/test_settings.py | 63 --- .../proxy/client/cli/test_auth_commands.py | 40 +- .../proxy/client/cli/test_claude_settings.py | 535 +++++++++++++++++- .../client/cli/test_configure_commands.py | 331 +++++++++++ .../test_litellm/proxy/client/cli/test_pi.py | 26 +- .../proxy/client/cli/test_up_commands.py | 40 +- 17 files changed, 1718 insertions(+), 250 deletions(-) delete mode 100644 litellm/proxy/client/cli/commands/autoroute/settings.py create mode 100644 litellm/proxy/client/cli/commands/configure.py delete mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_settings.py create mode 100644 tests/test_litellm/proxy/client/cli/test_configure_commands.py diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index f7d9eb7da9a..03d01ff7f66 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -508,7 +508,7 @@ The credential is short-lived by design (default 24h, configurable via `LITELLM_ ### Route Every Claude Code Session Through the Proxy -`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when those keys are missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. +`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when those keys are missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. Two things need to already be true: you've run `lite login` (or `lite login --pkce`, whose key the helper renews on its own), since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you. @@ -532,12 +532,28 @@ Cursor is not supported: it has no equivalent file-based config to hot-patch thi lite --base-url https://your-proxy.example.com login --config-claude ``` -It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY`, and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. +It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY`, and `apiKeyHelper`, but persistently: no foreground process to keep alive, and `lite unconfigure claude` restores what it changed (see below). Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. Because the credential is reached through `apiKeyHelper` rather than copied into the file, a later `lite login` refreshes it with no further action: Claude Code re-runs the helper on every request and picks up whatever token the most recent login stored. Nothing secret is written to `settings.json`. Run it again to point Claude Code at a different proxy; the base URL and the helper are both rewritten. `lite up` and `--config-claude` manage the same file, so the flag refuses to run while a `lite up` session holds a backup, and tells you to run `lite down` first, rather than writing settings that `lite up` would silently revert when it stops. +#### Configuring Claude Code Once, With a Virtual Key or Your Login + +`lite configure claude` wires Claude Code up persistently and `lite unconfigure claude` puts things back. It is what `lite login --config-claude` does, plus a pinned model and an undo, and it also takes a long-lived virtual key when that is what you have: + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh +lite --base-url https://your-proxy.example.com configure claude --api-key sk-... --model claude-auto +claude +``` + +With `--api-key` (or `lite --api-key` / `LITELLM_PROXY_API_KEY`) the key is written into `env.ANTHROPIC_AUTH_TOKEN`. Without one, your `lite login` credential is used the way `--config-claude` uses it, through `apiKeyHelper`, so a later `lite login` (or a `--pkce` renewal) picks up on its own and nothing secret lands in the file; a missing or stale login is refreshed first. Either way the command checks the key against `GET /v1/models`, then patches `~/.claude/settings.json`: `env.ANTHROPIC_BASE_URL`, the credential, and `env.ENABLE_TOOL_SEARCH` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` when those are missing, so Claude Code's `/model` picker lists the proxy's models (the ones whose id contains `claude` or `anthropic`) and you pick between them as usual. Claude Code keeps its own default model until you switch, so that id has to exist on the proxy for the first message to go through; `--model` (or the interactive prompt below) sets the model Claude Code starts on instead, as the top-level `model` key, which has to be on `/v1/models` for the key. Nothing forces Claude Code's sub-agent or background tiers onto a proxy model, so those built-in ids need to exist on the proxy too; `lite autoroute up` is the mode that pins every tier to one group. Claude Code treats a name it does not know as an unknown model: it prints a one-line `unrecognized_model` note, assumes a 200k context window and sends no thinking parameters for it, so either name the group like a Claude model id or append `[1m]` to opt into the 1M window. The other credential slots (`env.ANTHROPIC_API_KEY`, a stale `env.ANTHROPIC_AUTH_TOKEN` or `apiKeyHelper`) are removed so they cannot fight the one written. Every other setting is preserved and the file is written atomically with owner-only permissions; if `settings.json` is a symlink into a dotfiles repository, the key is written through to that target and the command says so, so keep it out of version control + +Plain `lite configure`, with no agent named, asks the same things interactively: which agents to wire (Claude Code today) and which of the proxy's models to start on, picked from `/v1/models` with a type-to-filter prompt + +What the command changed is recorded in `~/.litellm/claude_configure_state.json` (previous values plus fingerprints of what was written, never a second copy of the key). `lite unconfigure claude` restores each of those keys only if it still holds what `configure` wrote, so anything you changed since is left alone and named in the output; a `settings.json` or `env` object that only existed because of `configure` is removed again. Ownership moves only by a write: running `configure` again (a re-login is one) refreshes the record only for the keys its merge changed, keeps the original snapshot of a key that still holds what it wrote, and snapshots afresh a key you changed in between, so `unconfigure` brings back whatever the repeat displaced and never adopts your edit as its own. A credential (`env.ANTHROPIC_API_KEY`, `env.ANTHROPIC_AUTH_TOKEN`, `apiKeyHelper`) is put back only when the restored file points at the `ANTHROPIC_BASE_URL` it was captured next to; otherwise it stays removed, the output says which server it belonged to, and the receipt is kept so pointing the URL back and running `unconfigure` again finishes the job. It also undoes `lite login --config-claude`, which writes through the same path. Like `--config-claude`, both refuse to run while a `lite up` or `lite autoroute up` session holds a backup, and that check comes before any login prompt or request + ### QA Complexity-Based Auto-Routing Against Your Real Proxy `lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session. @@ -584,7 +600,7 @@ An interactive wizard. It runs the same model-group discovery as above, splits t The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key. -You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, and `ANTHROPIC_DEFAULT_OPUS_MODEL` to `autorouter` in `~/.claude/settings.json`, so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.) +You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets the top-level `model` and `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, `ANTHROPIC_DEFAULT_OPUS_MODEL` and `ANTHROPIC_DEFAULT_FABLE_MODEL` to `autorouter` in `~/.claude/settings.json` (and `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when missing, like every other wiring), so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.) You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first. diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 7a0ae9dc955..b287293eb7a 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -16,6 +16,7 @@ from .cmd_quoting import quote_for_cmd from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, + ListingFailure, PiSyncError, fetch_model_ids, fetch_model_limits, @@ -165,7 +166,9 @@ def prepare_pi( """ ids: Final = fetch_model_ids(base_url, api_key, get=get) if isinstance(ids, PiSyncError): - raise AgentRunError(ids.message) + raise AgentRunError( + f"{ids.message} pi would have nothing to run." if ids.kind is ListingFailure.EMPTY else ids.message + ) limits: Final = fetch_model_limits(base_url, api_key, get=get) path: Final = models_json_path(base_env) error: Final = sync_models_json(path, base_url, ids, limits) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 12a288202b6..24b1c766664 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -41,9 +41,15 @@ from litellm.litellm_core_utils.cli_token_utils import ( from .claude_settings import ( CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS, + STARTING_MODEL_ROLE, + ApiKeyHelper, ClaudeSettingsError, - write_claude_settings, + KeepModel, + configure_claude_settings, + refuse_while_owned, + resolve_api_key_helper, ) from .pkce_login import ( Http, @@ -778,13 +784,23 @@ def _render_and_prompt_for_team_selection(teams: list[CliTeam]) -> str | None: def _configure_claude_code(base_url: str) -> None: - """Point Claude Code at base_url by patching ~/.claude/settings.json.""" + """Point Claude Code at base_url by patching ~/.claude/settings.json, undoable with `lite unconfigure claude`.""" try: - write_claude_settings(base_url, CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + configure_claude_settings( + base_url, + ApiKeyHelper(resolve_api_key_helper(base_url)), + KeepModel(), + CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, + SETTINGS_FILE_OWNERS, + ) except ClaudeSettingsError as e: raise click.ClickException(f"Logged in, but could not configure Claude Code: {e}") click.echo(f"\nConfigured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url.rstrip('/')}.") - click.echo("Your other Claude Code settings were left untouched. Restart Claude Code to pick this up.") + click.echo( + "Your other Claude Code settings were left untouched. Restart Claude Code to pick this up. " + f"Undo with `lite unconfigure claude`; `lite configure claude --model` sets {STARTING_MODEL_ROLE}." + ) def _finish_login(base_url: str, api_key: str, config_claude: bool, stored: SecretSave) -> None: @@ -853,6 +869,11 @@ def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"] + if config_claude: + try: + refuse_while_owned(CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + except ClaudeSettingsError as e: + raise click.ClickException(f"Cannot configure Claude Code, so not logging in: {e}") try: if pkce: diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 26d45138a27..86701f186fd 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -14,11 +14,13 @@ from ..claude_settings import ( AUTOROUTE_BACKUP_PATH, CLAUDE_SETTINGS_PATH, ClaudeSettingsError, + StaticToken, load_json_or_empty, + merge_claude_settings, ) from ..up import BackupRecord as ClaudeBackupRecord from ..up import restore_claude_settings, write_backup -from .config import master_key_from_config +from .config import AUTOROUTER_MODEL_NAME, master_key_from_config from .process import ( CONFIG_PATH, DEFAULT_AUTOROUTE_PORT, @@ -37,7 +39,6 @@ from .process import ( terminate, write_pid_record, ) -from .settings import merge_claude_settings_static_token from .wizard import run_configure_wizard _GENERATED_CONFIG_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) @@ -156,7 +157,9 @@ def up(port: int) -> None: ClaudeBackupRecord(existed=original_existed, content=original_settings if original_existed else None), AUTOROUTE_BACKUP_PATH, ) - merged: Final = merge_claude_settings_static_token(original_settings, base_url, master_key) + merged: Final = merge_claude_settings( + original_settings, base_url, StaticToken(master_key), AUTOROUTER_MODEL_NAME, AUTOROUTER_MODEL_NAME + ) CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) with secure_create(CLAUDE_SETTINGS_PATH) as f: json.dump(merged, f, indent=2) diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py deleted file mode 100644 index 60729b5410d..00000000000 --- a/litellm/proxy/client/cli/commands/autoroute/settings.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Final - -from pydantic import JsonValue - -from .config import AUTOROUTER_MODEL_NAME - -ENV_KEY: Final = "env" -API_KEY_HELPER_KEY: Final = "apiKeyHelper" -ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" -ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN" -ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" -ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" -ENABLE_TOOL_SEARCH_VALUE: Final = "true" -# Force every one of Claude Code's own model tiers to request the auto-router by name. -# Router's auto-router registry is keyed by the literal requested model string -# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*" -# model_name can never work as a catch-all -- these overrides are what actually makes -# Claude Code send "autorouter" regardless of /model or its own version-specific defaults. -ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: Final = ( - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", -) - - -def merge_claude_settings_static_token( - settings: dict[str, JsonValue], base_url: str, auth_token: str -) -> dict[str, JsonValue]: - """Return a new settings dict wired to a local ephemeral proxy with a static token. - - Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real - remote proxy needing refreshable SSO tokens), this proxy is ephemeral and its key is the - locally persisted autoroute master key, so a plain env var is simpler and correct. Any - existing apiKeyHelper is cleared so it can't fight with the static token. - """ - raw_env: Final = settings.get(ENV_KEY, {}) - base_env: Final = raw_env if isinstance(raw_env, dict) else {} - env: Final[dict[str, JsonValue]] = { - ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, - **base_env, - ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), - ANTHROPIC_AUTH_TOKEN_KEY: auth_token, - **{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS}, - } - env.pop(ANTHROPIC_API_KEY_KEY, None) - merged: Final[dict[str, JsonValue]] = {**settings, ENV_KEY: env} - merged.pop(API_KEY_HELPER_KEY, None) - return merged - - -__all__ = ["merge_claude_settings_static_token"] diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index 5e3ce95f088..bd9cf85dc59 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -1,37 +1,70 @@ """Shared handling of Claude Code's ~/.claude/settings.json. -`lite up` patches this file temporarily and restores it on exit; `lite login ---config-claude` patches it persistently. Both need the same merge and the same -apiKeyHelper command, and `up` already imports from `auth`, so the shared parts -live here rather than in either command module. +`lite up` and `lite autoroute up` patch this file temporarily and restore it on +exit; `lite login --config-claude` and `lite configure claude` patch it +persistently and record how to undo it. All of them need the same merge and the +same apiKeyHelper command, and `up` already imports from `auth`, so the shared +parts live here rather than in any one command module. """ +import hashlib +import json import shlex import shutil import sys -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from functools import reduce +from itertools import chain from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, TypeAlias -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError -from litellm.litellm_core_utils.private_json import write_private_json +from litellm.litellm_core_utils.private_json import ( + commit_staged_json, + discard_staged_json, + ensure_private_dir, + stage_private_json, +) from .cmd_quoting import quote_for_cmd ENV_KEY: Final = "env" API_KEY_HELPER_KEY: Final = "apiKeyHelper" +MODEL_KEY: Final = "model" ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" +ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" ENABLE_TOOL_SEARCH_VALUE: Final = "true" ENABLE_GATEWAY_MODEL_DISCOVERY_KEY: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY" ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE: Final = "1" +ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: Final = ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_FABLE_MODEL", +) +OWNED_ENV_KEYS: Final = ( + ENABLE_TOOL_SEARCH_KEY, + ENABLE_GATEWAY_MODEL_DISCOVERY_KEY, + ANTHROPIC_BASE_URL_KEY, + ANTHROPIC_AUTH_TOKEN_KEY, + ANTHROPIC_API_KEY_KEY, +) +OWNED_TOP_LEVEL_KEYS: Final = (API_KEY_HELPER_KEY, MODEL_KEY) +OWNED_PATHS: Final = (*(f"{ENV_KEY}.{key}" for key in OWNED_ENV_KEYS), *OWNED_TOP_LEVEL_KEYS) +_CREDENTIAL_ENV_KEYS: Final = frozenset((ANTHROPIC_API_KEY_KEY, ANTHROPIC_AUTH_TOKEN_KEY)) +_CREDENTIAL_PATHS: Final = (*(f"{ENV_KEY}.{key}" for key in sorted(_CREDENTIAL_ENV_KEYS)), API_KEY_HELPER_KEY) +_BASE_URL_PATH: Final = f"{ENV_KEY}.{ANTHROPIC_BASE_URL_KEY}" +STARTING_MODEL_ROLE: Final = "the /model picker's default row, the model Claude Code starts on" CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json" +CONFIGURE_STATE_PATH: Final = Path.home() / ".litellm" / "claude_configure_state.json" @dataclass(frozen=True, slots=True) @@ -55,6 +88,102 @@ class ClaudeSettingsError(Exception): """Raised for any user-actionable failure while reading or writing Claude Code settings.""" +@dataclass(frozen=True, slots=True) +class StaticToken: + """A long-lived virtual key, written into env.ANTHROPIC_AUTH_TOKEN.""" + + token: str + + +@dataclass(frozen=True, slots=True) +class ApiKeyHelper: + """A `lite auth print-token` command Claude Code runs per request, so a login renews in place.""" + + command: str + + +ClaudeCredential: TypeAlias = StaticToken | ApiKeyHelper + + +@dataclass(frozen=True, slots=True) +class KeepModel: + """Leave the top-level `model` as it is, the user's or an earlier configure's (a re-login).""" + + +@dataclass(frozen=True, slots=True) +class UnpinModel: + """Let go of a `model` an earlier configure pinned; one the user set themselves stays.""" + + +@dataclass(frozen=True, slots=True) +class StartOn: + """Pin the top-level `model`, the row Claude Code starts on.""" + + model: str + + +ModelChoice: TypeAlias = KeepModel | UnpinModel | StartOn + + +class OwnedValue(BaseModel): + """What one key held at a moment in time; `present=False` is an absent key, not a null one.""" + + model_config = ConfigDict(frozen=True) + + present: bool + value: JsonValue = None + + +class ConfigureReceipt(BaseModel): + """What `lite configure claude` found and what it owns, keyed by dotted path (`env.X` or a top-level key). + + Ownership moves only by a write: `written` fingerprints the keys some configure changed, at the + value it wrote; a repeat configure refreshes a fingerprint only for a key its merge changed and + carries the earlier one otherwise, so a key the user edited in between stops matching and is left + alone. `previous` is what each key held before configure took it over; a repeat keeps the earlier + snapshot while the key still holds our value and snapshots afresh otherwise, so whatever the + repeat displaces is what comes back. `endpoints` is the ANTHROPIC_BASE_URL each credential slot + was captured beside, so a credential is only ever put back next to the server it was issued for. + No fingerprint is a second copy of a token. + """ + + model_config = ConfigDict(frozen=True) + + file_existed: bool + env_present: bool + env_was_object: bool + previous: Mapping[str, OwnedValue] + written: Mapping[str, str] + endpoints: Mapping[str, OwnedValue] + + +@dataclass(frozen=True, slots=True) +class WithheldCredential: + """A credential left removed: captured beside `endpoint`, while the restored file points elsewhere.""" + + key: str + endpoint: str + + +@dataclass(frozen=True, slots=True) +class UnconfigureOutcome: + """Keys whose value unconfigure changed back, keys the user changed since and so were left as they + are, credentials withheld (the receipt is kept for them, so a later unconfigure can finish once the + URL points back), and whether no settings file remains.""" + + restored: tuple[str, ...] + kept: tuple[str, ...] + withheld: tuple[WithheldCredential, ...] = () + file_removed: bool = False + + +@dataclass(frozen=True, slots=True) +class _Claim: + previous: OwnedValue + written: str | None + endpoint: OwnedValue | None + + def load_json_or_empty(path: Path) -> dict[str, JsonValue]: try: content: Final = path.read_bytes() if path.exists() else b"" @@ -70,29 +199,104 @@ def load_json_or_empty(path: Path) -> dict[str, JsonValue]: ) -def merge_claude_settings( - settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str -) -> dict[str, JsonValue]: - """Return a new settings dict wired to route Claude Code through the proxy. +def _env_object(settings: Mapping[str, JsonValue], path: Path) -> Mapping[str, JsonValue]: + raw_env: Final = settings.get(ENV_KEY) + if raw_env is None: + return MappingProxyType({}) + if not isinstance(raw_env, dict): + raise ClaudeSettingsError( + f'{path} has a non-object "{ENV_KEY}" value, which this would discard. Fix or remove it, then retry.' + ) + return raw_env - Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a - stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued - token (same reasoning as build_agent_env in agents.py). ENABLE_TOOL_SEARCH - defaults to true because Claude Code turns tool search off when - ANTHROPIC_BASE_URL is not a first-party Anthropic host, and - CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY defaults to 1 so the /model picker - is filled from the proxy's /v1/models; existing values of both are left - alone. Every other key is preserved untouched. + +def refuse_while_owned(settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: + """Refuse while `lite up` or `lite autoroute up` holds a backup it will restore over any write; a + purely local check, so commands run it before any login prompt or request.""" + for owner in owners: + if owner.backup_path.exists(): + raise ClaudeSettingsError( + f"`{owner.start_command}` is currently managing {settings_path} (backup at " + f"{owner.backup_path}) and will restore it when it stops. " + f"Run `{owner.stop_command}` first, then retry." + ) + + +def _write_target(settings_path: Path) -> Path: + """Write through a symlinked settings.json rather than replacing the link, which would silently + detach a file symlinked into a dotfiles repo.""" + try: + return settings_path.resolve() if settings_path.is_symlink() else settings_path + except OSError as e: + raise ClaudeSettingsError(f"Could not resolve {settings_path}: {e}") from e + + +def _stage(path: Path, document: Mapping[str, object]) -> str: + try: + return stage_private_json(str(path), document) + except OSError as e: + raise ClaudeSettingsError(f"Could not write {path}: {e}") from e + + +def _land( + path: Path, + staged: str | None, + also_discard: Sequence[str | None] = (), + commit: Callable[[str, str], None] = commit_staged_json, +) -> None: + """Commit a staged file to `path`, or remove `path` when nothing is staged for it. The one place a + filesystem error becomes a ClaudeSettingsError; on failure the operation's other staged files are + discarded, so no temp file holding a token is left behind.""" + try: + if staged is None: + path.unlink(missing_ok=True) + else: + commit(staged, str(path)) + except OSError as e: + for other in also_discard: + if other is not None: + discard_staged_json(other) + raise ClaudeSettingsError(f"Could not {'remove' if staged is None else 'write'} {path}: {e}") from e + + +def merge_claude_settings( + settings: Mapping[str, JsonValue], + base_url: str, + credential: ClaudeCredential, + default_model: str | None = None, + tier_model: str | None = None, +) -> Mapping[str, JsonValue]: + """Return a new settings mapping wired to route Claude Code through the proxy. + + A StaticToken lands in env.ANTHROPIC_AUTH_TOKEN, an ApiKeyHelper in the top-level apiKeyHelper; + the other credential slots are removed either way, since Claude Code given two credentials may + send the wrong one. ENABLE_TOOL_SEARCH and CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY get their + defaults only when missing. `default_model` is the top-level `model`, the row Claude Code starts + on; `tier_model` is `lite autoroute up`'s knob that points every ANTHROPIC_DEFAULT_*_MODEL at one + group. Apart from those tier keys, exactly OWNED_PATHS are touched. """ raw_env: Final = settings.get(ENV_KEY, {}) - base_env: Final = raw_env if isinstance(raw_env, dict) else {} - env: Final = { - ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, - ENABLE_GATEWAY_MODEL_DISCOVERY_KEY: ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE, - **{key: value for key, value in base_env.items() if key != ANTHROPIC_API_KEY_KEY}, - ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), - } - return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} + current_env: Final = raw_env if isinstance(raw_env, dict) else {} + env: Final = dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain( + ( + (ENABLE_TOOL_SEARCH_KEY, ENABLE_TOOL_SEARCH_VALUE), + (ENABLE_GATEWAY_MODEL_DISCOVERY_KEY, ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE), + ), + ((key, value) for key, value in current_env.items() if key not in _CREDENTIAL_ENV_KEYS), + ((ANTHROPIC_BASE_URL_KEY, base_url.rstrip("/")),), + ((ANTHROPIC_AUTH_TOKEN_KEY, credential.token),) if isinstance(credential, StaticToken) else (), + ((key, tier_model) for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS if tier_model is not None), + ) + ) + return dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain( + ((key, value) for key, value in settings.items() if key not in (API_KEY_HELPER_KEY, ENV_KEY)), + ((ENV_KEY, env),), + ((API_KEY_HELPER_KEY, credential.command),) if isinstance(credential, ApiKeyHelper) else (), + ((MODEL_KEY, default_model),) if default_model is not None else (), + ) + ) def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str: @@ -121,56 +325,255 @@ def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str: return " ".join(quote(token) for token in (lite_path, "--base-url", base_url, "auth", "print-token")) -def write_claude_settings(base_url: str, settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: - """Persistently point Claude Code at base_url, preserving every unrelated setting. +def _owned(container: Mapping[str, JsonValue], key: str) -> OwnedValue: + return OwnedValue(present=key in container, value=container.get(key)) - Refuses while any owner holds a backup: each restores its backup when it - stops, which would silently undo this write. - """ - for owner in owners: - if owner.backup_path.exists(): - raise ClaudeSettingsError( - f"`{owner.start_command}` is currently managing {settings_path} (backup at " - f"{owner.backup_path}) and will restore it when it stops. " - f"Run `{owner.stop_command}` first, then retry." - ) - normalized_base_url: Final = base_url.rstrip("/") - api_key_helper: Final = resolve_api_key_helper(normalized_base_url) - existing: Final = load_json_or_empty(settings_path) - raw_env: Final = existing.get(ENV_KEY) - if raw_env is not None and not isinstance(raw_env, dict): - raise ClaudeSettingsError( - f'{settings_path} has a non-object "{ENV_KEY}" value, which this would discard. ' - "Fix or remove it, then retry." - ) - merged: Final = merge_claude_settings(existing, normalized_base_url, api_key_helper) - # os.replace() swaps the symlink itself for a regular file, silently detaching a - # settings.json that is symlinked into a dotfiles repo. There is no backup to undo - # that here, unlike `lite up`, so write through to the link's target instead. - target: Final = settings_path.resolve() if settings_path.is_symlink() else settings_path + +def _fingerprint(owned: OwnedValue) -> str: + return hashlib.sha256(json.dumps(owned.model_dump(mode="json"), sort_keys=True).encode()).hexdigest() + + +def _env(settings: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + raw_env: Final = settings.get(ENV_KEY) + return raw_env if isinstance(raw_env, dict) else MappingProxyType({}) + + +def _lookup(settings: Mapping[str, JsonValue], path: str) -> OwnedValue: + section, _, key = path.rpartition(".") + return _owned(_env(settings) if section else settings, key) + + +def _with_key(container: Mapping[str, JsonValue], key: str, owned: OwnedValue) -> Mapping[str, JsonValue]: + return dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain(((k, v) for k, v in container.items() if k != key), ((key, owned.value),) if owned.present else ()) + ) + + +def _with(settings: Mapping[str, JsonValue], path: str, owned: OwnedValue) -> Mapping[str, JsonValue]: + """`settings` with the key at `path` set (or removed when `owned` is absent); nothing else changes.""" + section, _, key = path.rpartition(".") + if not section: + return _with_key(settings, key, owned) + return _with_key(settings, section, OwnedValue(present=True, value=_with_key(_env(settings), key, owned))) + + +def _with_all(settings: Mapping[str, JsonValue], updates: Mapping[str, OwnedValue]) -> Mapping[str, JsonValue]: + return reduce(lambda acc, item: _with(acc, *item), updates.items(), settings) + + +def _ours(settings: Mapping[str, JsonValue], path: str, receipt: ConfigureReceipt) -> bool: + """Whether the key still holds what a configure wrote (a key no configure ever changed is never ours).""" + return receipt.written.get(path) == _fingerprint(_lookup(settings, path)) + + +def _claim( + path: str, + current: Mapping[str, JsonValue], + merged: Mapping[str, JsonValue], + earlier: ConfigureReceipt | None, + url_now: OwnedValue, +) -> _Claim: + """What this configure records for one key; see ConfigureReceipt for the rules.""" + before, after = _lookup(current, path), _lookup(merged, path) + carried: Final = earlier if earlier is not None and _ours(current, path, earlier) else None + return _Claim( + previous=before if carried is None else carried.previous.get(path, before), + written=_fingerprint(after) if before != after else (None if earlier is None else earlier.written.get(path)), + endpoint=None + if path not in _CREDENTIAL_PATHS + else (url_now if carried is None else carried.endpoints.get(path, url_now)), + ) + + +def _receipt( + current: Mapping[str, JsonValue], + merged: Mapping[str, JsonValue], + earlier: ConfigureReceipt | None, + file_exists: bool, +) -> ConfigureReceipt: + url_now: Final = _lookup(current, _BASE_URL_PATH) + claims: Final = MappingProxyType({path: _claim(path, current, merged, earlier, url_now) for path in OWNED_PATHS}) + return ConfigureReceipt( + file_existed=file_exists if earlier is None else earlier.file_existed, + env_present=ENV_KEY in current if earlier is None else earlier.env_present, + env_was_object=isinstance(current.get(ENV_KEY), dict) if earlier is None else earlier.env_was_object, + previous=MappingProxyType({path: claim.previous for path, claim in claims.items()}), + written=MappingProxyType({path: claim.written for path, claim in claims.items() if claim.written is not None}), + endpoints=MappingProxyType( + {path: claim.endpoint for path, claim in claims.items() if claim.endpoint is not None} + ), + ) + + +def read_configure_receipt(state_path: Path) -> ConfigureReceipt | None: + if not state_path.exists(): + return None try: - write_private_json(str(target), merged) + return ConfigureReceipt.model_validate_json(state_path.read_bytes()) + except (OSError, ValidationError) as e: + raise ClaudeSettingsError( + f"{state_path} is not a readable `lite configure claude` receipt ({e}). " + "Remove it and edit Claude Code's settings by hand if they still point at the proxy." + ) from e + + +def configure_claude_settings( + base_url: str, + credential: ClaudeCredential, + model: ModelChoice, + settings_path: Path, + state_path: Path, + owners: Sequence[SettingsFileOwner], + commit: Callable[[str, str], None] = commit_staged_json, +) -> None: + """Persistently route Claude Code through base_url, recording how to undo it. + + Both files are staged before either is committed, so a full disk or a read-only directory fails + before anything changes. The two commits are still two renames: a receipt rename that fails + discards the staged settings, and a settings rename that fails after the receipt landed puts the + earlier receipt back (or removes the new one), so the receipt on disk never describes settings + that were not written. `model`: StartOn pins the starting model, UnpinModel lets go of a pin an + earlier configure made (never of the user's own), KeepModel leaves it alone (a re-login). + """ + refuse_while_owned(settings_path, owners) + current: Final = load_json_or_empty(settings_path) + _env_object(current, settings_path) + earlier: Final = read_configure_receipt(state_path) + existing: Final = ( + _with(current, MODEL_KEY, earlier.previous[MODEL_KEY]) + if isinstance(model, UnpinModel) and earlier is not None and _ours(current, MODEL_KEY, earlier) + else current + ) + merged: Final = merge_claude_settings( + existing, base_url, credential, model.model if isinstance(model, StartOn) else None + ) + receipt: Final = _receipt(current, merged, earlier, settings_path.exists()) + target: Final = _write_target(settings_path) + try: + ensure_private_dir(state_path.parent) except OSError as e: - raise ClaudeSettingsError(f"Could not write {target}: {e}") from e + raise ClaudeSettingsError(f"Could not write {state_path}: {e}") from e + staged_receipt: Final = _stage(state_path, receipt.model_dump(mode="json")) + try: + staged_settings: Final = _stage(target, merged) + except ClaudeSettingsError: + discard_staged_json(staged_receipt) + raise + _land(state_path, staged_receipt, (staged_settings,), commit) + try: + _land(target, staged_settings, commit=commit) + except ClaudeSettingsError as settings_error: + try: + _land(state_path, None if earlier is None else _stage(state_path, earlier.model_dump(mode="json"))) + except ClaudeSettingsError as receipt_error: + raise ClaudeSettingsError( + f"{settings_error} The receipt at {state_path} now describes settings that were not written and " + f"could not be put back either ({receipt_error}); remove it before retrying." + ) from settings_error + raise + + +def _endpoint_text(endpoint: OwnedValue) -> str: + if not endpoint.present: + return f"no {ANTHROPIC_BASE_URL_KEY} (Anthropic's default endpoint)" + return endpoint.value if isinstance(endpoint.value, str) else json.dumps(endpoint.value) + + +def unconfigure_claude_settings( + settings_path: Path, state_path: Path, owners: Sequence[SettingsFileOwner] +) -> UnconfigureOutcome: + """Undo `lite configure claude`: put back every key still holding what configure wrote, leave the + rest alone, and withhold a credential the restored file would send to a different server than it + was issued for (the receipt stays, owning only those slots, so a later unconfigure can finish).""" + refuse_while_owned(settings_path, owners) + receipt: Final = read_configure_receipt(state_path) + if receipt is None: + raise ClaudeSettingsError( + f"Claude Code is not configured by `lite configure claude` (no receipt at {state_path}); nothing to undo." + ) + current: Final = load_json_or_empty(settings_path) + _env_object(current, settings_path) + ours: Final = tuple(path for path in receipt.written if _ours(current, path, receipt)) + kept: Final = tuple(path for path in receipt.written if path not in ours and _lookup(current, path).present) + put_back: Final = _with_all(current, MappingProxyType({path: receipt.previous[path] for path in ours})) + url_after: Final = _lookup(put_back, _BASE_URL_PATH) + withheld: Final = tuple( + WithheldCredential(path, _endpoint_text(receipt.endpoints[path])) + for path in _CREDENTIAL_PATHS + if path in ours and receipt.previous[path].present and receipt.endpoints[path] != url_after + ) + absent: Final = OwnedValue(present=False) + trimmed: Final = _with_all(put_back, MappingProxyType({item.key: absent for item in withheld})) + settings: Final = ( + trimmed + if _env(trimmed) or receipt.env_was_object + else _with_key(trimmed, ENV_KEY, OwnedValue(present=receipt.env_present, value=None)) + ) + target: Final = _write_target(settings_path) + file_removed: Final = not settings and not (receipt.file_existed and target.exists()) + kept_receipt: Final = ( # mutable-ok: pydantic serializes the update as given and rejects a mappingproxy + receipt.model_copy(update={"written": {item.key: _fingerprint(absent) for item in withheld}}) + if withheld + else None + ) + staged_settings: Final = None if file_removed else _stage(target, settings) + try: + staged_receipt: Final = ( + None if kept_receipt is None else _stage(state_path, kept_receipt.model_dump(mode="json")) + ) + except ClaudeSettingsError: + if staged_settings is not None: + discard_staged_json(staged_settings) + raise + _land(target, staged_settings, (staged_receipt,)) + _land(state_path, staged_receipt) + return UnconfigureOutcome( + restored=tuple(path for path in ours if _lookup(current, path) != _lookup(settings, path)), + kept=kept, + withheld=withheld, + file_removed=file_removed, + ) __all__ = ( "ANTHROPIC_API_KEY_KEY", + "ANTHROPIC_AUTH_TOKEN_KEY", "ANTHROPIC_BASE_URL_KEY", + "ANTHROPIC_DEFAULT_MODEL_ENV_KEYS", "API_KEY_HELPER_KEY", "AUTOROUTE_BACKUP_PATH", "BACKUP_PATH", "CLAUDE_SETTINGS_PATH", + "CONFIGURE_STATE_PATH", "ENABLE_GATEWAY_MODEL_DISCOVERY_KEY", "ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE", "ENABLE_TOOL_SEARCH_KEY", "ENABLE_TOOL_SEARCH_VALUE", "ENV_KEY", + "MODEL_KEY", + "OWNED_ENV_KEYS", + "OWNED_PATHS", + "OWNED_TOP_LEVEL_KEYS", "SETTINGS_FILE_OWNERS", + "STARTING_MODEL_ROLE", + "ApiKeyHelper", + "ClaudeCredential", "ClaudeSettingsError", + "ConfigureReceipt", + "KeepModel", + "ModelChoice", + "OwnedValue", "SettingsFileOwner", + "StartOn", + "StaticToken", + "UnconfigureOutcome", + "UnpinModel", + "WithheldCredential", + "configure_claude_settings", "load_json_or_empty", "merge_claude_settings", + "read_configure_receipt", + "refuse_while_owned", "resolve_api_key_helper", - "write_claude_settings", + "unconfigure_claude_settings", ) diff --git a/litellm/proxy/client/cli/commands/configure.py b/litellm/proxy/client/cli/commands/configure.py new file mode 100644 index 00000000000..f3a98a48476 --- /dev/null +++ b/litellm/proxy/client/cli/commands/configure.py @@ -0,0 +1,252 @@ +"""`lite configure claude` and `lite unconfigure claude`: persistent Claude Code wiring, undoable.""" + +import re +import sys +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Final + +import click +from InquirerPy import inquirer +from InquirerPy.base.control import Choice + +from .auth import CliContextObj, context_secret_vault, get_stored_api_key +from .claude_settings import ( + CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, + SETTINGS_FILE_OWNERS, + STARTING_MODEL_ROLE, + ApiKeyHelper, + ClaudeCredential, + ClaudeSettingsError, + ModelChoice, + StartOn, + StaticToken, + UnconfigureOutcome, + UnpinModel, + configure_claude_settings, + refuse_while_owned, + resolve_api_key_helper, + unconfigure_claude_settings, +) +from .pi import ListingFailure, PiSyncError, fetch_model_ids +from .up import ensure_fresh_login + +_LISTED_MODELS_SHOWN: Final = 20 +_CLAUDE_TARGET: Final = "claude" +_TARGETS: Final = ((_CLAUDE_TARGET, "Claude Code (CLI)"),) +_KEEP_DEFAULT_MODEL: Final = "Keep Claude Code's own default" +_CLAUDE_CODE_PICKER_FILTER: Final = re.compile(r"claude|anthropic", re.IGNORECASE) +_MODEL_OPTION_HELP: Final = ( + f"Proxy model to set as {STARTING_MODEL_ROLE}. Must be listed on /v1/models for the key; without it, " + "Claude Code keeps its own default and a pin an earlier configure made is let go of. Nothing pins Claude " + "Code's sub-agent or background tiers; `lite autoroute up` is the mode that does." +) + + +def resolve_credential(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, str]: + """The credential to write and the key to check the proxy with. + + An explicit key (--api-key, `lite --api-key`, LITELLM_PROXY_API_KEY) is long-lived and goes + into settings.json as a static token. Without one, the stored `lite login` credential is used + the way `lite login --config-claude` uses it, through apiKeyHelper, since it expires within a + day and renews in place there; a missing or stale login is refreshed first, as `lite up` does. + """ + ctx_obj: Final[CliContextObj] = ctx.obj + explicit: Final = api_key or (None if ctx_obj.get("api_key_from_token_file") else ctx_obj.get("api_key")) + if explicit: + return StaticToken(explicit), explicit + base_url: Final = ctx_obj["base_url"] + ensure_fresh_login(ctx) + stored: Final = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) + if not stored: + raise ClaudeSettingsError("Login did not produce a usable token.") + return ApiKeyHelper(resolve_api_key_helper(base_url)), stored + + +def _start(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, tuple[str, ...]]: + """Every configure path begins the same way: the local ownership check first, so a `lite up` + session is refused before any login prompt or request, then the credential, then the listing.""" + try: + refuse_while_owned(CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + credential, key = resolve_credential(ctx, api_key) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + return credential, _listed_models(ctx.obj["base_url"], key) + + +def _listing_error(base_url: str, error: PiSyncError) -> str: + """The hint that fits how the listing failed: only an unreachable proxy gets the "is it running" question.""" + if error.kind is ListingFailure.REJECTED: + return f"LiteLLM rejected your key (HTTP {error.status}). Run `lite login` to refresh it, or pass a valid --api-key." + if error.kind is ListingFailure.UNREACHABLE: + return f"{error.message} Is the proxy at {base_url} running, and is --base-url (or LITELLM_PROXY_URL) correct?" + if error.kind is ListingFailure.EMPTY: + return f"{error.message} Claude Code would have nothing to run; give the key access to at least one model." + return f"{error.message} The proxy at {base_url} answered, so check that it is a LiteLLM proxy and is healthy." + + +def _listed_models(base_url: str, key: str) -> tuple[str, ...]: + listed: Final = fetch_model_ids(base_url, key) + if isinstance(listed, PiSyncError): + raise click.ClickException(_listing_error(base_url, listed)) + return listed + + +def _model_choice(model: str | None) -> ModelChoice: + return StartOn(model) if model is not None else UnpinModel() + + +def _apply_claude(ctx: click.Context, credential: ClaudeCredential, listed: Sequence[str], model: str | None) -> None: + ctx_obj: Final[CliContextObj] = ctx.obj + base_url: Final = ctx_obj["base_url"] + if model is not None and model not in listed: + shown: Final = ", ".join(listed[:_LISTED_MODELS_SHOWN]) + more: Final = f", and {len(listed) - _LISTED_MODELS_SHOWN} more" if len(listed) > _LISTED_MODELS_SHOWN else "" + raise click.ClickException( + f"{model!r} is not served by {base_url} for this key. /v1/models lists: {shown}{more}." + ) + try: + configure_claude_settings( + base_url, credential, _model_choice(model), CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS + ) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + in_picker: Final = sum(1 for listed_model in listed if _CLAUDE_CODE_PICKER_FILTER.search(listed_model)) + click.echo(f"Configured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url}.") + click.echo( + "Credential: your virtual key, stored in the file as ANTHROPIC_AUTH_TOKEN." + if isinstance(credential, StaticToken) + else "Credential: your `lite login`, read through apiKeyHelper on every request, so a later login renews it." + ) + click.echo( + f"Starting model: {model} ({STARTING_MODEL_ROLE}); switch any time with /model." + if model is not None + else "Starting model: not pinned (Claude Code's default, or a model you set yourself); switch with /model, or " + "pass --model to start on a proxy model." + ) + click.echo( + f"/model will list {in_picker} of the proxy's {len(listed)} models (Claude Code shows only ids containing " + "'claude' or 'anthropic')." + ) + click.echo("Start `claude` from any terminal. Undo with `lite unconfigure claude`.") + if isinstance(credential, StaticToken) and CLAUDE_SETTINGS_PATH.is_symlink(): + click.echo( + f"Note: {CLAUDE_SETTINGS_PATH} is a symlink to {CLAUDE_SETTINGS_PATH.resolve()}, so your key now lives in " + "that file; keep it out of version control.", + err=True, + ) + + +def _pick_targets() -> tuple[str, ...]: + picked: Final = inquirer.checkbox( + message="Which agents should route through LiteLLM?", + choices=[Choice(value, name=label, enabled=True) for value, label in _TARGETS], + validate=lambda chosen: len(chosen) > 0, + invalid_message="Pick at least one.", + ).execute() + return tuple(str(value) for value in picked) + + +def _pick_model(listed: Sequence[str]) -> str | None: + picked: Final = inquirer.fuzzy( + message="Model Claude Code starts on (type to filter; /model switches any time):", + choices=[_KEEP_DEFAULT_MODEL, *listed], + ).execute() + return None if picked == _KEEP_DEFAULT_MODEL else str(picked) + + +def interactive_configure( + ctx: click.Context, + pick_targets: Callable[[], tuple[str, ...]] = _pick_targets, + pick_model: Callable[[Sequence[str]], str | None] = _pick_model, +) -> None: + """`lite configure` with no agent named: ask which agents to wire and which model to pin.""" + targets: Final = pick_targets() + if _CLAUDE_TARGET not in targets: + return + credential, listed = _start(ctx, None) + _apply_claude(ctx, credential, listed, pick_model(listed)) + + +@click.group(name="configure", invoke_without_command=True) +@click.pass_context +def configure_group(ctx: click.Context) -> None: + """Persistently route a coding agent through your LiteLLM proxy. + + With no agent named, asks which agents to wire and which proxy model to pin. + """ + if ctx.invoked_subcommand is not None: + return + if not sys.stdin.isatty(): + raise click.ClickException( + "`lite configure` asks questions, so it needs a terminal. Non-interactively, run " + "`lite configure claude --api-key --model `." + ) + interactive_configure(ctx) + + +@click.group(name="unconfigure") +def unconfigure_group() -> None: + """Undo `lite configure` for a coding agent.""" + + +@configure_group.command(name="claude") +@click.option( + "--api-key", + "api_key", + default=None, + help="Long-lived LiteLLM virtual key written into Claude Code's settings. Defaults to the `lite --api-key` / " + "LITELLM_PROXY_API_KEY value; with neither, your `lite login` credential is used through apiKeyHelper.", +) +@click.option("--model", default=None, help=_MODEL_OPTION_HELP) +@click.pass_context +def configure_claude(ctx: click.Context, api_key: str | None, model: str | None) -> None: + """Route every Claude Code session through your LiteLLM proxy until `lite unconfigure claude`. + + Patches ~/.claude/settings.json in place: the proxy URL, your credential (a virtual key as a + static token, or your `lite login` through apiKeyHelper), and gateway model discovery so + /model lists the proxy's models; --model picks the one Claude Code starts on. Every other + setting is kept, and what changed is recorded so `lite unconfigure claude` can put it back. + Assumes the proxy is already running. + """ + credential, listed = _start(ctx, api_key) + _apply_claude(ctx, credential, listed, model) + + +@unconfigure_group.command(name="claude") +def unconfigure_claude() -> None: + """Return Claude Code's settings to what they were before `lite configure claude`. + + Also undoes `lite login --config-claude`. Only keys still holding what configure wrote are + put back; anything you changed since is left as it is and named in the output. + """ + try: + outcome: Final = unconfigure_claude_settings(CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + _report_unconfigure(CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, outcome) + + +def _report_unconfigure(settings_path: Path, state_path: Path, outcome: UnconfigureOutcome) -> None: + """Say what unconfigure did, naming only keys whose value it changed.""" + if outcome.file_removed: + click.echo( + f"No settings file remains at {settings_path}; it held nothing but `lite configure claude`'s own keys." + ) + elif outcome.restored: + click.echo(f"Restored in {settings_path}: {', '.join(outcome.restored)}.") + else: + click.echo(f"Nothing in {settings_path} was still ours to restore.") + if outcome.kept: + click.echo(f"Left as you changed them since: {', '.join(outcome.kept)}.") + if outcome.withheld: + click.echo( + "Left removed, since the file now points at a different server than they were issued for: " + + "; ".join(f"{item.key} (captured with {item.endpoint})" for item in outcome.withheld) + + f". They stay in {state_path}: point env.ANTHROPIC_BASE_URL back and run `lite unconfigure claude` " + "again to put them back, or delete that file to drop them." + ) + + +__all__ = ("configure_group", "interactive_configure", "resolve_credential", "unconfigure_group") diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 7b0c1970c4e..70c89a853e3 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -10,6 +10,7 @@ import os import tempfile from collections.abc import Callable, Mapping from dataclasses import dataclass +from enum import StrEnum from pathlib import Path from types import MappingProxyType from typing import Final @@ -20,11 +21,28 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError PI_CONFIG_DIR_ENV: Final = "PI_CODING_AGENT_DIR" PI_PROVIDER_NAME: Final = "litellm" LITELLM_PROXY_API_KEY_ENV: Final = "LITELLM_PROXY_API_KEY" +_REJECTED_STATUSES: Final = frozenset((401, 403)) + + +class ListingFailure(StrEnum): + """Why a proxy could not be listed, decided once where the HTTP outcome is classified. + + `unreachable` means no response at all; the other kinds prove the proxy answered, so callers + must not suggest checking whether it is running. + """ + + UNREACHABLE = "unreachable" + REJECTED = "rejected" + BAD_BODY = "bad_body" + EMPTY = "empty" + OTHER = "other" @dataclass(frozen=True, slots=True) class PiSyncError: message: str + status: int | None = None + kind: ListingFailure | None = None @dataclass(frozen=True, slots=True) @@ -65,16 +83,20 @@ def fetch_model_ids( timeout=10, ) except requests.RequestException as e: - return PiSyncError(f"Could not list models from the proxy: {e}") + return PiSyncError(f"Could not list models from the proxy: {e}", kind=ListingFailure.UNREACHABLE) if resp.status_code != 200: - return PiSyncError(f"The proxy returned HTTP {resp.status_code} for /v1/models; cannot build pi's model list.") + return PiSyncError( + f"The proxy returned HTTP {resp.status_code} for /v1/models; cannot list models.", + resp.status_code, + ListingFailure.REJECTED if resp.status_code in _REJECTED_STATUSES else ListingFailure.OTHER, + ) try: listing: Final = _ModelList.model_validate(resp.json()) except (ValueError, ValidationError) as e: - return PiSyncError(f"Unexpected /v1/models response from the proxy: {e}") + return PiSyncError(f"Unexpected /v1/models response from the proxy: {e}", kind=ListingFailure.BAD_BODY) ids: Final = tuple(dict.fromkeys(model.id for model in listing.data)) if not ids: - return PiSyncError("The proxy returned no models for your key, so pi would have nothing to run.") + return PiSyncError("The proxy returned no models for your key.", kind=ListingFailure.EMPTY) return ids @@ -200,6 +222,7 @@ __all__ = ( "LITELLM_PROXY_API_KEY_ENV", "PI_CONFIG_DIR_ENV", "PI_PROVIDER_NAME", + "ListingFailure", "ModelLimits", "PiSyncError", "fetch_model_ids", diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index b7c02866d6f..ffece87ab83 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -23,6 +23,7 @@ from .auth import CliContextObj, context_secret_vault, get_stored_api_key, load_ from .claude_settings import ( BACKUP_PATH, CLAUDE_SETTINGS_PATH, + ApiKeyHelper, ClaudeSettingsError, load_json_or_empty, merge_claude_settings, @@ -123,7 +124,7 @@ def _stored_login_is_pkce(vault: SecretVault) -> bool: return token_data is not None and token_data.get("refresh_token") is not None -def _ensure_fresh_login(ctx: click.Context) -> None: +def ensure_fresh_login(ctx: click.Context) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"].rstrip("/") vault: Final = context_secret_vault(ctx) @@ -141,7 +142,7 @@ def _ensure_fresh_login(ctx: click.Context) -> None: click.echo("No fresh LiteLLM login found for this proxy; starting login...") ctx.invoke(login, pkce=pkce) if not _usable_login(get_stored_api_key(expected_base_url=base_url, vault=vault), vault): - raise UpError("Login did not produce a usable token; cannot start `lite up`.") + raise UpError("Login did not produce a usable token.") def _restore_and_report() -> None: @@ -169,7 +170,7 @@ def up(ctx: click.Context) -> None: base_url: Final = ctx.obj["base_url"] try: - _ensure_fresh_login(ctx) + ensure_fresh_login(ctx) api_key: Final = resolve_api_key(ctx) verify_proxy_key(base_url, api_key) @@ -190,7 +191,7 @@ def up(ctx: click.Context) -> None: ) CLAUDE_SETTINGS_PATH.parent.mkdir(exist_ok=True) - merged: Final = merge_claude_settings(original_settings, base_url, api_key_helper) + merged: Final = merge_claude_settings(original_settings, base_url, ApiKeyHelper(api_key_helper)) with open(CLAUDE_SETTINGS_PATH, "w") as f: json.dump(merged, f, indent=2) except (AgentRunError, ClaudeSettingsError) as e: diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index eae1b0f5bc9..b0e81a222c0 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -13,6 +13,7 @@ from .commands.auth import auth_group, context_secret_vault, get_stored_api_key, from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names +from .commands.configure import configure_group, unconfigure_group from .commands.credentials import credentials from .commands.debug import debug from .commands.encryption import encryption @@ -162,6 +163,9 @@ cli.add_command(model_groups) # Add the autoroute command group (QA auto-routing against your real proxy) cli.add_command(autoroute_group, name="autoroute") cli.add_command(config_commands) +# Add configure/unconfigure (persistently wire a coding agent to the proxy with a virtual key) +cli.add_command(configure_group) +cli.add_command(unconfigure_group) if __name__ == "__main__": diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 4a3b3ef22c4..77742ea9f9f 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -159,6 +159,10 @@ class TestUpCommand: assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key" assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true" assert "apiKeyHelper" not in captured["settings"] + # The ephemeral proxy serves only the autorouter, so a starting model left by + # `lite configure claude --model` or a user pin would 400 on the first message. + assert captured["settings"]["model"] == "autorouter" + assert captured["settings"]["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "autorouter" assert captured["settings_mode"] == 0o600 assert terminate_calls == [99999] diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py deleted file mode 100644 index 87a33c79a79..00000000000 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py +++ /dev/null @@ -1,63 +0,0 @@ -from litellm.proxy.client.cli.commands.autoroute.settings import ( - ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, - merge_claude_settings_static_token, -) - - -def test_preserves_unrelated_top_level_keys(): - merged = merge_claude_settings_static_token({"theme": "dark"}, "http://127.0.0.1:4000", "token-abc") - assert merged["theme"] == "dark" - - -def test_preserves_unrelated_env_keys(): - settings = {"env": {"SOME_OTHER_VAR": "value"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["SOME_OTHER_VAR"] == "value" - - -def test_sets_base_url_and_auth_token(): - merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000/", "token-abc") - assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" - assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" - assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" - - -def test_preserves_existing_tool_search(): - settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" - - -def test_drops_stray_api_key(): - settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert "ANTHROPIC_API_KEY" not in merged["env"] - - -def test_removes_existing_api_key_helper(): - settings = {"apiKeyHelper": "/usr/local/bin/lite auth print-token"} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert "apiKeyHelper" not in merged - - -def test_does_not_mutate_input(): - settings = {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} - merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert settings == {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} - - -def test_forces_all_claude_code_default_model_tiers_to_the_autorouter(): - # A bare "*" model_name deployment looks like the obvious way to catch every request - # regardless of which model Claude Code thinks it's using, but Router's auto-router - # registry is keyed by the literal requested model string with no wildcard resolution - # (litellm/router.py:10711-10717) -- so the only reliable way to make every one of Claude - # Code's own tiers hit the auto-router is to override the env vars it reads per tier. - merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000", "token-abc") - for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: - assert merged["env"][key] == "autorouter" - - -def test_overrides_a_preexisting_default_model_env_var(): - settings = {"env": {"ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-opus-4-8"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "autorouter" diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 821323e722c..7d5cb61c062 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -27,6 +27,7 @@ from litellm.proxy.client.cli.commands.auth import ( print_token, whoami, ) +from litellm.proxy.client.cli.commands import auth as auth_module from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner @@ -1398,8 +1399,10 @@ class TestLoginConfigClaude: def setup_method(self): self.runner = CliRunner() - def _run_login(self, tmp_path, args, base_url="https://test.example.com"): + def _run_login(self, tmp_path, monkeypatch, args, base_url="https://test.example.com"): settings_path = tmp_path / "claude" / "settings.json" + monkeypatch.setattr(auth_module, "CLAUDE_SETTINGS_PATH", settings_path) + monkeypatch.setattr(auth_module, "CONFIGURE_STATE_PATH", tmp_path / "claude_configure_state.json") backup_path = tmp_path / "claude_settings_backup.json" poll_response = Mock() poll_response.status_code = 200 @@ -1416,7 +1419,6 @@ class TestLoginConfigClaude: patch("requests.get", return_value=poll_response), patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), - patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), patch( "litellm.proxy.client.cli.commands.auth.SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),), @@ -1429,16 +1431,16 @@ class TestLoginConfigClaude: result = self.runner.invoke(login, args, obj={"base_url": base_url}) return result, settings_path, backup_path - def test_default_login_does_not_touch_claude_settings(self, tmp_path): - result, settings_path, _backup_path = self._run_login(tmp_path, []) + def test_default_login_does_not_touch_claude_settings(self, tmp_path, monkeypatch): + result, settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, []) assert result.exit_code == 0 assert "Login successful!" in result.output assert not settings_path.exists() assert "Configured Claude Code" not in result.output - def test_flag_writes_the_settings_file_and_reports_success(self, tmp_path): - result, settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + def test_flag_writes_the_settings_file_and_reports_success(self, tmp_path, monkeypatch): + result, settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code == 0 written = json.loads(settings_path.read_text()) @@ -1446,25 +1448,43 @@ class TestLoginConfigClaude: assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" assert "Configured Claude Code" in result.output + assert "pins a proxy model for every tier" not in result.output + assert "the model Claude Code starts on" in result.output - def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path): + def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path, monkeypatch): settings_path = tmp_path / "claude" / "settings.json" settings_path.parent.mkdir(parents=True) settings_path.write_text(json.dumps({"theme": "dark", "env": {"KEEP": "me"}})) - result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + result, _settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code == 0 written = json.loads(settings_path.read_text()) assert written["theme"] == "dark" assert written["env"]["KEEP"] == "me" - def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path): + def test_refuses_before_logging_in_while_lite_up_holds_the_settings(self, tmp_path, monkeypatch): + # The local precondition comes first: no browser, no token stored, no "Login successful!". + backup_path = tmp_path / "claude_settings_backup.json" + backup_path.write_text("{}") + monkeypatch.setattr(auth_module, "CLAUDE_SETTINGS_PATH", tmp_path / "claude" / "settings.json") + monkeypatch.setattr( + auth_module, "SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),) + ) + with patch("requests.post") as post, patch("webbrowser.open") as browser: + result = self.runner.invoke(login, ["--config-claude"], obj={"base_url": "https://test.example.com"}) + assert result.exit_code != 0 + assert "not logging in" in result.output and "lite down" in result.output + assert "Login successful!" not in result.output + post.assert_not_called() + browser.assert_not_called() + + def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path, monkeypatch): settings_path = tmp_path / "claude" / "settings.json" settings_path.parent.mkdir(parents=True) settings_path.write_text("not json at all {{{") - result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + result, _settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code != 0 assert "Login successful!" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index e5f2a9d95bd..9072fe26b00 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -1,4 +1,5 @@ import json +import os import shlex import stat import time @@ -9,14 +10,25 @@ from click.testing import CliRunner from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli import cli +from litellm.litellm_core_utils.private_json import commit_staged_json from litellm.proxy.client.cli.commands.claude_settings import ( + ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, AUTOROUTE_BACKUP_PATH, BACKUP_PATH, + OWNED_ENV_KEYS, + OWNED_TOP_LEVEL_KEYS, SETTINGS_FILE_OWNERS, + ApiKeyHelper, ClaudeSettingsError, + KeepModel, SettingsFileOwner, + StartOn, + StaticToken, + UnpinModel, + configure_claude_settings, + merge_claude_settings, resolve_api_key_helper, - write_claude_settings, + unconfigure_claude_settings, ) @@ -24,6 +36,7 @@ def _owners(*backup_paths): """Stand-in owners for the real `lite up` / `lite autoroute up` registry.""" return tuple(SettingsFileOwner(path, "lite up", "lite down") for path in backup_paths) + CLAUDE_SETTINGS_MODULE = "litellm.proxy.client.cli.commands.claude_settings" AUTH_MODULE = "litellm.proxy.client.cli.commands.auth" WINDOWS_LITE_EXE = "C:\\Users\\u\\AppData\\Local\\Programs\\Python\\Python313\\Scripts\\lite.EXE" @@ -97,17 +110,28 @@ def lite_on_path(): yield -class TestWriteClaudeSettings: +def _helper_configure(base_url, settings_path, owners, state_path=None): + """`lite login --config-claude`'s shape: the login credential behind apiKeyHelper, no pinned model.""" + state = state_path if state_path is not None else settings_path.parent.parent / "state.json" + root = base_url.rstrip("/") + configure_claude_settings( + root, ApiKeyHelper(resolve_api_key_helper(root)), KeepModel(), settings_path, state, owners + ) + + +class TestConfigureWithTheLoginHelper: def test_creates_the_file_and_its_parent_when_missing(self, paths, lite_on_path): settings_path, backup_path = paths assert not settings_path.parent.exists() - write_claude_settings("https://proxy.example.com/", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com/", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" + assert written["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token" + assert "model" not in written def test_updates_an_existing_file_preserving_unrelated_settings(self, paths, lite_on_path): settings_path, backup_path = paths @@ -123,7 +147,7 @@ class TestWriteClaudeSettings: ) ) - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["theme"] == "dark" @@ -135,26 +159,31 @@ class TestWriteClaudeSettings: def test_rerunning_against_a_new_proxy_refreshes_both_base_url_and_helper(self, paths, lite_on_path): settings_path, backup_path = paths - write_claude_settings("https://first.example.com", settings_path, _owners(backup_path)) - write_claude_settings("https://second.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://first.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://second.example.com", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://second.example.com" assert "second.example.com" in written["apiKeyHelper"] assert "first.example.com" not in written["apiKeyHelper"] - def test_drops_a_stray_static_api_key_so_the_helper_token_wins(self, paths, lite_on_path): + def test_drops_stray_static_credentials_so_the_helper_token_wins(self, paths, lite_on_path): + # Claude Code prefers ANTHROPIC_AUTH_TOKEN over apiKeyHelper, so a virtual key left behind + # by an earlier `lite configure claude --api-key` would silently keep winning. settings_path, backup_path = paths settings_path.parent.mkdir(parents=True) - settings_path.write_text(json.dumps({"env": {"ANTHROPIC_API_KEY": "sk-leaked"}})) + settings_path.write_text( + json.dumps({"env": {"ANTHROPIC_API_KEY": "sk-leaked", "ANTHROPIC_AUTH_TOKEN": "sk-old"}}) + ) - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) - assert "ANTHROPIC_API_KEY" not in json.loads(settings_path.read_text())["env"] + env = json.loads(settings_path.read_text())["env"] + assert "ANTHROPIC_API_KEY" not in env and "ANTHROPIC_AUTH_TOKEN" not in env def test_written_file_is_owner_only(self, paths, lite_on_path): settings_path, backup_path = paths - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert stat.S_IMODE(settings_path.stat().st_mode) == 0o600 def test_refuses_while_lite_up_holds_a_backup(self, paths, lite_on_path): @@ -162,7 +191,7 @@ class TestWriteClaudeSettings: backup_path.write_text("{}") with pytest.raises(ClaudeSettingsError, match="lite down"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert not settings_path.exists() @@ -172,7 +201,7 @@ class TestWriteClaudeSettings: settings_path.write_text("not json at all {{{") with pytest.raises(ClaudeSettingsError, match="invalid JSON"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert settings_path.read_text() == "not json at all {{{" @@ -180,7 +209,7 @@ class TestWriteClaudeSettings: settings_path, backup_path = paths with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value=None): with pytest.raises(ClaudeSettingsError, match="Could not find `lite`"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert not settings_path.exists() @@ -196,7 +225,7 @@ class TestWriteClaudeSettings: settings_path.write_bytes(b'{"theme": "\xff\xfe"}') with pytest.raises(ClaudeSettingsError, match="invalid JSON"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) def test_reports_an_actionable_error_when_the_file_cannot_be_read(self, paths, lite_on_path): """An unreadable settings file must not surface as "Authentication failed". @@ -210,16 +239,18 @@ class TestWriteClaudeSettings: settings_path.mkdir() with pytest.raises(ClaudeSettingsError, match="Could not read"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) def test_reports_an_actionable_error_when_the_file_cannot_be_written(self, paths, lite_on_path): settings_path, backup_path = paths - with patch( - f"{CLAUDE_SETTINGS_MODULE}.write_private_json", - side_effect=OSError("Read-only file system"), - ): - with pytest.raises(ClaudeSettingsError, match="Read-only file system"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + settings_path.parent.mkdir(parents=True) + settings_path.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not write"): + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) + finally: + settings_path.parent.chmod(0o700) + assert not settings_path.exists() class TestApiKeyHelperIsActuallyInvocable: @@ -303,7 +334,7 @@ class TestConflictingOwnersOfTheSettingsFile: backup.write_text("{}") stand_in = SettingsFileOwner(backup, owner.start_command, owner.stop_command) with pytest.raises(ClaudeSettingsError, match="currently managing"): - write_claude_settings("https://proxy.example.com", settings_path, (stand_in,)) + _helper_configure("https://proxy.example.com", settings_path, (stand_in,)) backup.unlink() assert not settings_path.exists() @@ -314,9 +345,9 @@ class TestConflictingOwnersOfTheSettingsFile: autoroute = SettingsFileOwner(backup, "lite autoroute up", "lite autoroute down") with pytest.raises(ClaudeSettingsError, match="`lite autoroute up` is currently managing"): - write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + _helper_configure("https://proxy.example.com", settings_path, (autoroute,)) with pytest.raises(ClaudeSettingsError, match="Run `lite autoroute down` first"): - write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + _helper_configure("https://proxy.example.com", settings_path, (autoroute,)) def test_the_registry_matches_the_paths_the_commands_actually_use(self): """A second definition of the autoroute dir must not drift from this one.""" @@ -341,7 +372,7 @@ class TestDoesNotDestroyUserOwnedStructure: link.parent.mkdir() link.symlink_to(real) - write_claude_settings("https://proxy.example.com", link, ()) + _helper_configure("https://proxy.example.com", link, ()) assert link.is_symlink() assert json.loads(real.read_text())["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" @@ -354,6 +385,456 @@ class TestDoesNotDestroyUserOwnedStructure: settings_path.write_text(json.dumps({"theme": "dark", "env": "not-an-object"})) with pytest.raises(ClaudeSettingsError, match="non-object"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert json.loads(settings_path.read_text())["env"] == "not-an-object" + + +class TestMergeClaudeSettings: + """One merge for every way Claude Code gets wired: `lite up`, `lite login --config-claude`, + `lite configure claude` and `lite autoroute up`.""" + + def test_a_static_token_lands_in_env_and_the_helper_slot_is_cleared(self): + settings = {"apiKeyHelper": "/usr/local/bin/lite auth print-token", "env": {"ANTHROPIC_API_KEY": "leaked"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000/", StaticToken("token-abc")) + assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" + assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" + assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" + assert "ANTHROPIC_API_KEY" not in merged["env"] + assert "apiKeyHelper" not in merged + assert "model" not in merged + assert not any(key in merged["env"] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS) + + def test_a_helper_lands_top_level_and_the_static_slots_are_cleared(self): + settings = {"env": {"ANTHROPIC_AUTH_TOKEN": "sk-old", "ANTHROPIC_API_KEY": "leaked"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", ApiKeyHelper("lite auth print-token")) + assert merged["apiKeyHelper"] == "lite auth print-token" + assert "ANTHROPIC_AUTH_TOKEN" not in merged["env"] and "ANTHROPIC_API_KEY" not in merged["env"] + + def test_keeps_existing_switch_values_and_unrelated_keys_without_mutating_the_input(self): + settings = {"theme": "dark", "env": {"SOME_OTHER_VAR": "value", "ENABLE_TOOL_SEARCH": "false"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", StaticToken("token-abc")) + assert merged["theme"] == "dark" + assert merged["env"]["SOME_OTHER_VAR"] == "value" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" + assert settings == {"theme": "dark", "env": {"SOME_OTHER_VAR": "value", "ENABLE_TOOL_SEARCH": "false"}} + + def test_a_default_model_sets_only_the_row_claude_code_starts_on(self): + merged = merge_claude_settings( + {}, "http://127.0.0.1:4000", StaticToken("token-abc"), default_model="claude-auto" + ) + assert merged["model"] == "claude-auto" + assert not any(key in merged["env"] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS) + + def test_a_tier_model_forces_every_claude_code_tier_as_autoroute_needs(self): + # Router's auto-router registry is keyed by the literal requested model string with no + # wildcard resolution, so `lite autoroute up` overrides the env var each tier reads. + settings = {"env": {"ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-opus-4-8"}} + merged = merge_claude_settings( + settings, "http://127.0.0.1:4000", StaticToken("token-abc"), tier_model="autorouter" + ) + assert {merged["env"][key] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS} == {"autorouter"} + assert "model" not in merged + + def test_touches_exactly_the_declared_owned_keys(self): + # The receipt and unconfigure restore exactly OWNED_*_KEYS, so a key the merge writes outside + # that table would be written by configure and never undone. + settings = { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"KEEP_ME": "1", "ANTHROPIC_API_KEY": "old", "ENABLE_TOOL_SEARCH": "false"}, + "apiKeyHelper": "old-helper", + "model": "old-model", + } + for credential in (StaticToken("token-abc"), ApiKeyHelper("helper")): + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", credential, default_model="claude-auto") + changed_top_level = {key for key in set(settings) | set(merged) if settings.get(key) != merged.get(key)} + assert changed_top_level - {"env"} <= set(OWNED_TOP_LEVEL_KEYS) + changed_env = { + key + for key in set(settings["env"]) | set(merged["env"]) + if settings["env"].get(key) != merged["env"].get(key) + } + assert changed_env <= set(OWNED_ENV_KEYS) + assert merged["permissions"] == {"allow": ["Bash"]} + assert merged["env"]["KEEP_ME"] == "1" + + +PROXY = "http://127.0.0.1:4000" +ANTHROPIC = "https://api.anthropic.com" +HELPER = ApiKeyHelper("lite auth print-token") +ORIGINAL = { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"KEEP_ME": "1", "ANTHROPIC_API_KEY": "sk-ant-mine", "ANTHROPIC_BASE_URL": ANTHROPIC}, + "apiKeyHelper": "/usr/local/bin/lite auth print-token", + "model": "claude-opus-5", +} + + +def _set(path, value): + """A user edit: set (or with `_ABSENT`, remove) the key at a dotted path in the settings file.""" + + def edit(settings): + section, _, key = path.rpartition(".") + container = settings.setdefault(section, {}) if section else settings + if value is _ABSENT: + container.pop(key, None) + else: + container[key] = value + return settings + + return edit + + +_ABSENT = object() + + +class _Rig: + """One settings file plus receipt under tmp_path, driven through the public functions only.""" + + def __init__(self, tmp_path, initial): + self.settings = tmp_path / "claude" / "settings.json" + self.state = tmp_path / "state" / "claude_configure_state.json" + if initial is not None: + self.settings.parent.mkdir(parents=True) + self.settings.write_text(json.dumps(initial)) + + def read(self): + return json.loads(self.settings.read_text()) if self.settings.exists() else None + + def configure(self, credential=StaticToken("sk-virtual-key"), model=StartOn("claude-auto"), **kwargs): + configure_claude_settings(PROXY, credential, model, self.settings, self.state, (), **kwargs) + + def edit(self, *edits): + settings = self.read() + for apply in edits: + settings = apply(settings) + self.settings.write_text(json.dumps(settings)) + + def unconfigure(self): + return unconfigure_claude_settings(self.settings, self.state, ()) + + +# Each row: initial file, steps (configure kwargs dicts or edit callables) between the first configure +# and unconfigure, the expected file afterwards, and the expected outcome fields. Sequences that used +# to be one test each; the receipt's rules are what make them all come out right. +UNDO_SCENARIOS = { + "plain round trip": (ORIGINAL, [], ORIGINAL, {"kept": ()}), + "no file before": (None, [], None, {"file_removed": True}), + "no env before": ({"theme": "dark"}, [], {"theme": "dark"}, {}), + "null env before": ({"theme": "dark", "env": None}, [], {"theme": "dark", "env": None}, {}), + "empty env before": ({"theme": "dark", "env": {}}, [], {"theme": "dark", "env": {}}, {}), + "user edits stay and are named": ( + ORIGINAL, + [_set("env.ENABLE_TOOL_SEARCH", "false"), _set("model", "claude-sonnet-4-6")], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "claude-sonnet-4-6"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}, "withheld": ()}, + ), + "user filled an env configure created": (None, [_set("env.MY_VAR", "mine")], {"env": {"MY_VAR": "mine"}}, {}), + "user deleted the file": (None, [lambda s: None], None, {"file_removed": True, "restored": (), "kept": ()}), + "user removed our key: neither restored nor kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_AUTH_TOKEN", _ABSENT)], + ORIGINAL, + {"not_restored": {"env.ANTHROPIC_AUTH_TOKEN"}, "kept": ()}, + ), + "restored names only what changed": ( + {"model": "claude-opus-5"}, + [], + {"model": "claude-opus-5"}, + { + "restored": { + "env.ANTHROPIC_BASE_URL", + "env.ENABLE_TOOL_SEARCH", + "env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY", + "apiKeyHelper", + }, + "kept": (), + }, + {"credential": HELPER, "model": KeepModel()}, + ), + "repeat across credential kinds keeps the first snapshot": ( + ORIGINAL, + [ + {"credential": HELPER, "model": UnpinModel()}, + {"credential": StaticToken("sk-rotated"), "model": StartOn("claude-sonnet-4-6")}, + ], + ORIGINAL, + {}, + ), + "repeat without a model lets go of our pin, user had none": ({}, [{"model": UnpinModel()}], {}, {}), + "repeat without a model lets go of our pin, user had one": ( + {"model": "claude-opus-5"}, + [{"model": UnpinModel()}], + {"model": "claude-opus-5"}, + {}, + ), + "re-login keeps our pin": (None, [{"credential": HELPER, "model": KeepModel()}], None, {"file_removed": True}), + "edit between configures survives an unpin repeat": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": HELPER, "model": UnpinModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures survives a re-login": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": HELPER, "model": KeepModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures: a same-model repeat displaces it, so it is what comes back": ( + ORIGINAL, + [_set("model", "my-favourite"), _set("env.ENABLE_TOOL_SEARCH", "false"), {"credential": HELPER}], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH"}, "restored_includes": {"model"}}, + ), + "base URL changed since: credentials withheld, receipt kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {**ORIGINAL, "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, "apiKeyHelper": _ABSENT}, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC), ("apiKeyHelper", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL"}, + "receipt_kept": True, + }, + ), + "base URL changed and back: judged against the URL the restored file holds": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", ANTHROPIC)], + ORIGINAL, + {"withheld": ()}, + ), + "credential captured beside no URL goes back only beside no URL": ( + {"env": {"ANTHROPIC_API_KEY": "sk-default-endpoint"}}, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {"env": {"ANTHROPIC_BASE_URL": "http://other-proxy:4000"}}, + { + "withheld": {("env.ANTHROPIC_API_KEY", "no ANTHROPIC_BASE_URL (Anthropic's default endpoint)")}, + "receipt_kept": True, + }, + ), + "restored document empty while a credential is withheld: file goes, receipt stays": ( + None, + [ + _set("env.ANTHROPIC_API_KEY", "sk-user"), + {"credential": HELPER, "model": KeepModel()}, + _set("env.ANTHROPIC_BASE_URL", _ABSENT), + ], + None, + {"withheld": {("env.ANTHROPIC_API_KEY", PROXY)}, "file_removed": True, "receipt_kept": True}, + {"credential": HELPER, "model": KeepModel()}, + ), + "a credential the user changed is kept, never also withheld": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000"), _set("apiKeyHelper", "/opt/mine/helper")], + { + **ORIGINAL, + "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, + "apiKeyHelper": "/opt/mine/helper", + }, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL", "apiKeyHelper"}, + "receipt_kept": True, + }, + ), +} + + +def _expected_file(expected): + if expected is None: + return None + return {k: v for k, v in expected.items() if v is not _ABSENT} + + +class TestConfigureAndUnconfigure: + """`configure_claude_settings` records how to undo itself; `unconfigure_claude_settings` undoes only that.""" + + @pytest.mark.parametrize("scenario", UNDO_SCENARIOS.values(), ids=UNDO_SCENARIOS.keys()) + def test_undo_matrix(self, tmp_path, scenario): + initial, steps, expected, outcome_expectations, *first = scenario + rig = _Rig(tmp_path, initial) + rig.configure(**(first[0] if first else {})) + for step in steps: + if isinstance(step, dict): + rig.configure(**step) + elif rig.settings.exists() and step(json.loads(rig.settings.read_text())) is None: + rig.settings.unlink() + else: + rig.edit(step) + + outcome = rig.unconfigure() + + assert rig.read() == _expected_file(expected) + assert rig.state.exists() == outcome_expectations.get("receipt_kept", False) + for field, want in outcome_expectations.items(): + if field == "withheld": + assert {(item.key, item.endpoint) for item in outcome.withheld} == set(want) + elif field == "not_restored": + assert not set(want) & set(outcome.restored) and not set(want) & set(outcome.kept) + elif field == "restored_includes": + assert set(want) <= set(outcome.restored) + elif field in ("restored", "kept"): + assert set(getattr(outcome, field)) == set(want) + elif field != "receipt_kept": + assert getattr(outcome, field) == want + assert not {item.key for item in outcome.withheld} & set(outcome.kept) + + def test_configure_writes_owner_only_and_the_receipt_never_holds_the_key(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure(credential=StaticToken("sk-virtual-key-never-on-disk-twice")) + configured = rig.read() + assert configured["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key-never-on-disk-twice" + assert configured["env"]["ANTHROPIC_BASE_URL"] == PROXY and configured["model"] == "claude-auto" + assert "ANTHROPIC_API_KEY" not in configured["env"] and "apiKeyHelper" not in configured + assert stat.S_IMODE(rig.settings.stat().st_mode) == 0o600 == stat.S_IMODE(rig.state.stat().st_mode) + assert "sk-virtual-key-never-on-disk-twice" not in rig.state.read_text() + + def test_withheld_credentials_come_back_once_the_url_points_at_their_server_again(self, tmp_path): + # The kept receipt owns only the withheld slots: the second unconfigure restores exactly those. + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + rig.edit(_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")) + rig.unconfigure() + rig.edit(_set("env.ANTHROPIC_BASE_URL", ANTHROPIC), _set("theme", "light")) + outcome = rig.unconfigure() + assert rig.read() == {**ORIGINAL, "theme": "light"} + assert set(outcome.restored) == {"env.ANTHROPIC_API_KEY", "apiKeyHelper"} + assert outcome.kept == () and outcome.withheld == () and not rig.state.exists() + + @pytest.mark.parametrize( + ("path", "value", "repeat_credential"), + [ + ("env.ANTHROPIC_API_KEY", "sk-user-added-later", HELPER), + ("env.ANTHROPIC_AUTH_TOKEN", "sk-users-own-token", HELPER), + ("apiKeyHelper", "/opt/mine/helper", StaticToken("sk-rotated")), + ], + ids=["user-adds-api-key", "user-replaces-our-token", "user-sets-own-helper"], + ) + def test_a_credential_the_user_set_between_two_configures_is_what_comes_back( + self, tmp_path, path, value, repeat_credential + ): + # The repeat's merge clears the slot, so the displaced value is snapshotted and is what returns; + # it was set while the file pointed at the proxy, so it returns once the file points there again. + rig = _Rig(tmp_path, {"theme": "dark"}) + rig.configure(credential=HELPER, model=KeepModel()) + rig.edit(_set(path, value)) + rig.configure(credential=repeat_credential, model=KeepModel()) + assert not _lookup(rig.read(), path) + + outcome = rig.unconfigure() + assert [(item.key, item.endpoint) for item in outcome.withheld] == [(path, PROXY)] + assert rig.read() == {"theme": "dark"} and rig.state.exists() + rig.settings.write_text(json.dumps({"theme": "dark", "env": {"ANTHROPIC_BASE_URL": PROXY}})) + outcome = rig.unconfigure() + assert _lookup(rig.read(), path) == value + assert outcome.restored == (path,) and outcome.withheld == () and not rig.state.exists() + + def test_a_receipt_commit_that_fails_leaves_no_staged_token_behind(self, tmp_path): + rig = _Rig(tmp_path, {}) + + def commit_receipt_fails(staged, path): + if path == str(rig.state): + os.unlink(staged) + raise OSError("receipt rename failed") + commit_staged_json(staged, path) + + with pytest.raises(ClaudeSettingsError, match=r"Could not write .*receipt rename failed"): + rig.configure(credential=StaticToken("sk-never-left-in-a-temp-file"), commit=commit_receipt_fails) + assert not list(rig.settings.parent.glob(".tmp-*")) and not list(rig.state.parent.glob(".tmp-*")) + assert rig.read() == {} and not rig.state.exists() + + @pytest.mark.parametrize("configured_before", [False, True], ids=["first-configure", "repeat-configure"]) + def test_a_settings_commit_that_fails_after_the_receipt_landed_puts_the_receipt_back( + self, tmp_path, configured_before + ): + # The two renames are not atomic: a settings rename that fails after the receipt landed must + # not leave a receipt describing settings that were never written. + rig = _Rig(tmp_path, ORIGINAL) + if configured_before: + rig.configure() + receipt_before = rig.state.read_text() if configured_before else None + settings_before = rig.settings.read_text() + + def commit_settings_fails(staged, path): + if path == str(rig.settings): + os.unlink(staged) + raise OSError("rename failed") + commit_staged_json(staged, path) + + with pytest.raises(ClaudeSettingsError, match="rename failed"): + rig.configure(credential=StaticToken("sk-rotated"), commit=commit_settings_fails) + assert rig.settings.read_text() == settings_before + assert (rig.state.read_text() if rig.state.exists() else None) == receipt_before + if configured_before: + rig.unconfigure() + assert rig.read() == ORIGINAL + + def test_a_failed_repeat_configure_leaves_the_earlier_undo_intact(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + receipt_before = rig.state.read_text() + rig.settings.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not write"): + rig.configure(credential=StaticToken("sk-rotated")) + finally: + rig.settings.parent.chmod(0o700) + assert rig.state.read_text() == receipt_before and not list(rig.state.parent.glob(".tmp-*")) + assert rig.read()["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key" + rig.unconfigure() + assert rig.read() == ORIGINAL + + def test_unconfigure_reports_a_receipt_it_cannot_remove_as_a_settings_error(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + rig.state.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not remove"): + rig.unconfigure() + finally: + rig.state.parent.chmod(0o700) + + def test_configure_writes_through_a_symlinked_settings_file(self, tmp_path): + target = tmp_path / "dotfiles" / "settings.json" + target.parent.mkdir() + target.write_text(json.dumps({"theme": "dark"})) + link = tmp_path / "settings.json" + link.symlink_to(target) + configure_claude_settings(PROXY, StaticToken("sk-virtual-key"), UnpinModel(), link, tmp_path / "state.json", ()) + assert link.is_symlink() + assert json.loads(target.read_text())["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key" + + @pytest.mark.parametrize("operation", ["configure", "unconfigure"]) + def test_refuses_while_a_temporary_owner_holds_a_backup(self, paths, tmp_path, operation): + settings_path, backup_path = paths + backup_path.write_text("{}") + owners = _owners(backup_path) + state = tmp_path / "state.json" + attempt = ( + (lambda: configure_claude_settings(PROXY, StaticToken("k"), UnpinModel(), settings_path, state, owners)) + if operation == "configure" + else (lambda: unconfigure_claude_settings(settings_path, state, owners)) + ) + with pytest.raises(ClaudeSettingsError, match="lite down"): + attempt() + assert not settings_path.exists() + + def test_unconfigure_without_a_receipt_is_an_error_not_a_silent_no_op(self, tmp_path): + with pytest.raises(ClaudeSettingsError, match="nothing to undo"): + _Rig(tmp_path, None).unconfigure() + + +def _lookup(settings, path): + section, _, key = path.rpartition(".") + return (settings.get(section) or {}).get(key) if section else settings.get(key) diff --git a/tests/test_litellm/proxy/client/cli/test_configure_commands.py b/tests/test_litellm/proxy/client/cli/test_configure_commands.py new file mode 100644 index 00000000000..1631ba7f49e --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_configure_commands.py @@ -0,0 +1,331 @@ +import json +import os +import stat + +import click +import pytest +import requests +import responses +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands import configure as configure_module +from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner +from litellm.proxy.client.cli.commands.configure import configure_claude, configure_group, interactive_configure + +PROXY = "http://proxy.test:4000" +VALID_KEY = "sk-virtual-key" +LISTED_MODELS = ("claude-auto", "gpt-5.6-luna") + + +def _mock_models(): + responses.get( + f"{PROXY}/v1/models", + json={"data": [{"id": model, "object": "model"} for model in LISTED_MODELS]}, + match=[responses.matchers.header_matcher({"Authorization": f"Bearer {VALID_KEY}"})], + ) + responses.get(f"{PROXY}/v1/models", status=401) + + +@pytest.fixture +def paths(monkeypatch, tmp_path): + settings_path = tmp_path / "claude" / "settings.json" + state_path = tmp_path / "litellm" / "claude_configure_state.json" + monkeypatch.setattr(configure_module, "CLAUDE_SETTINGS_PATH", settings_path) + monkeypatch.setattr(configure_module, "CONFIGURE_STATE_PATH", state_path) + return settings_path, state_path + + +@pytest.fixture +def lite_on_path(monkeypatch, tmp_path): + """A real `lite` executable on PATH, so the apiKeyHelper command resolves without patching.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + lite = bin_dir / "lite" + lite.write_text("#!/bin/sh\nexit 0\n") + lite.chmod(lite.stat().st_mode | stat.S_IXUSR) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") + return str(lite) + + +@pytest.fixture +def runner(): + return CliRunner() + + +@pytest.fixture +def lite_up_backup(monkeypatch, tmp_path): + """A `lite up` session holding its backup, the local precondition every settings write refuses on.""" + backup = tmp_path / "claude_settings_backup.json" + backup.write_text("{}") + monkeypatch.setattr(configure_module, "SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup, "lite up", "lite down"),)) + return backup + + +def _configure(runner, *args): + return runner.invoke(cli, ["--base-url", PROXY, "configure", "claude", *args]) + + +class TestConfigureClaudeWithAVirtualKey: + @responses.activate + def test_writes_settings_and_reports_without_echoing_the_key(self, runner, paths): + _mock_models() + settings_path, state_path = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto") + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_BASE_URL"] == PROXY + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + assert written["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" + assert written["model"] == "claude-auto" + assert "ANTHROPIC_DEFAULT_SONNET_MODEL" not in written["env"] + assert state_path.exists() + assert VALID_KEY not in result.output + assert "Starting model: claude-auto" in result.output + assert "1 of the proxy's 2 models" in result.output + assert "lite unconfigure claude" in result.output + assert len(responses.calls) == 1 + + @responses.activate + def test_takes_the_key_from_the_global_option_and_keeps_claude_codes_default(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = runner.invoke(cli, ["--base-url", PROXY, "--api-key", VALID_KEY, "configure", "claude"]) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + assert "model" not in written + assert "Starting model: not pinned" in result.output + + @responses.activate + def test_refuses_a_model_the_proxy_does_not_list(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", "claude-nope") + assert result.exit_code != 0 + assert "'claude-nope' is not served" in result.output + assert "claude-auto, gpt-5.6-luna" in result.output + assert not settings_path.exists() + + @responses.activate + def test_refuses_a_key_the_proxy_rejects(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = _configure(runner, "--api-key", "sk-wrong") + assert result.exit_code != 0 + assert "rejected your key (HTTP 401)" in result.output + assert not settings_path.exists() + + @responses.activate + @pytest.mark.parametrize( + ("mock", "expected", "unexpected"), + [ + ( + lambda: responses.get(f"{PROXY}/v1/models", body=requests.ConnectionError("refused")), + "Is the proxy at", + "answered", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", status=500), + "The proxy at http://proxy.test:4000 answered", + "Is the proxy at", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", body="not json"), + "answered, so check that it is a LiteLLM proxy", + "Is the proxy at", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", json={"data": []}), + "Claude Code would have nothing to run", + "Is the proxy at", + ), + ], + ids=["unreachable", "http-500", "non-json-body", "empty-list"], + ) + def test_the_listing_hint_matches_how_the_listing_failed(self, runner, paths, mock, expected, unexpected): + # Only a proxy that never answered gets the "is it running" question; a 500, a non-JSON body or an + # empty list prove it is up, and the hint says so instead. + mock() + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY) + assert result.exit_code != 0 + assert expected in result.output and unexpected not in result.output + assert not settings_path.exists() + + @responses.activate + @pytest.mark.parametrize("entry", ["virtual-key", "login", "interactive"]) + def test_refuses_while_lite_up_holds_a_backup_before_any_login_or_request( + self, runner, paths, monkeypatch, lite_up_backup, entry + ): + _mock_models() + + def login_must_not_run(ctx): + raise AssertionError("the local precondition must be checked before a login is attempted") + + monkeypatch.setattr(configure_module, "ensure_fresh_login", login_must_not_run) + if entry == "interactive": + ctx = click.Context(configure_group, obj={"base_url": PROXY, "api_key": None}) + with pytest.raises(click.ClickException, match="lite down"): + interactive_configure(ctx, pick_targets=lambda: ("claude",), pick_model=lambda listed: None) + else: + args = ["--api-key", VALID_KEY] if entry == "virtual-key" else [] + result = runner.invoke(configure_claude, args, obj={"base_url": PROXY, "api_key": None}) + assert result.exit_code != 0 and "lite down" in result.output + assert len(responses.calls) == 0 + assert not paths[0].exists() + + @responses.activate + def test_says_so_when_the_key_is_written_through_a_symlink(self, runner, paths, tmp_path): + _mock_models() + settings_path, _ = paths + target = tmp_path / "dotfiles" / "settings.json" + target.parent.mkdir() + target.write_text("{}") + settings_path.parent.mkdir(parents=True) + settings_path.symlink_to(target) + result = _configure(runner, "--api-key", VALID_KEY) + assert result.exit_code == 0, result.output + assert "keep it out of version control" in result.output + assert json.loads(target.read_text())["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + + +class TestConfigureClaudeWithTheLogin: + def _stored_login(self, monkeypatch): + monkeypatch.setattr(configure_module, "ensure_fresh_login", lambda ctx: None) + monkeypatch.setattr(configure_module, "get_stored_api_key", lambda expected_base_url, vault: VALID_KEY) + + @responses.activate + def test_uses_the_login_through_the_helper_and_writes_no_secret(self, runner, paths, monkeypatch, lite_on_path): + _mock_models() + self._stored_login(monkeypatch) + settings_path, _ = paths + result = runner.invoke( + configure_claude, + ["--model", "claude-auto"], + obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": True}, + ) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["apiKeyHelper"] == f"{lite_on_path} --base-url {PROXY} auth print-token" + assert "ANTHROPIC_AUTH_TOKEN" not in written["env"] + assert written["model"] == "claude-auto" + assert VALID_KEY not in settings_path.read_text() + assert "read through apiKeyHelper" in result.output + + @responses.activate + def test_an_explicit_key_still_wins_over_a_stored_login(self, runner, paths, monkeypatch, lite_on_path): + _mock_models() + self._stored_login(monkeypatch) + settings_path, _ = paths + result = runner.invoke( + configure_claude, + ["--api-key", VALID_KEY], + obj={"base_url": PROXY, "api_key": "sk-login-jwt", "api_key_from_token_file": True}, + ) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY and "apiKeyHelper" not in written + + +class TestInteractiveConfigure: + @responses.activate + def test_asks_for_targets_and_a_starting_model_then_configures(self, paths): + _mock_models() + settings_path, _ = paths + asked = {} + + def pick_model(listed): + asked["listed"] = tuple(listed) + return "claude-auto" + + ctx = click.Context( + configure_group, obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": False} + ) + interactive_configure(ctx, pick_targets=lambda: ("claude",), pick_model=pick_model) + assert asked["listed"] == LISTED_MODELS + assert json.loads(settings_path.read_text())["model"] == "claude-auto" + + def test_does_nothing_when_claude_code_is_not_picked(self, paths): + settings_path, _ = paths + ctx = click.Context( + configure_group, obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": False} + ) + interactive_configure(ctx, pick_targets=lambda: (), pick_model=lambda listed: None) + assert not settings_path.exists() + + def test_bare_configure_without_a_terminal_names_the_non_interactive_command(self, runner, paths): + result = runner.invoke(cli, ["--base-url", PROXY, "configure"]) + assert result.exit_code != 0 + assert "lite configure claude --api-key" in result.output + + +class TestUnconfigureClaude: + @responses.activate + def test_restores_the_original_file_and_removes_the_receipt(self, runner, paths): + _mock_models() + settings_path, state_path = paths + settings_path.parent.mkdir(parents=True) + original = {"theme": "dark", "model": "claude-opus-5"} + settings_path.write_text(json.dumps(original)) + assert _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto").exit_code == 0 + + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert json.loads(settings_path.read_text()) == original + assert not state_path.exists() + assert "Restored in" in result.output and "model" in result.output + assert "ANTHROPIC_API_KEY" not in result.output, "a key that never existed was not restored" + + @responses.activate + def test_a_file_only_configure_created_is_reported_removed_not_restored(self, runner, paths): + _mock_models() + settings_path, _ = paths + assert _configure(runner, "--api-key", VALID_KEY).exit_code == 0 + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert not settings_path.exists() + assert "No settings file remains" in result.output and "Restored" not in result.output + + @responses.activate + def test_says_when_nothing_was_still_ours_and_names_what_it_kept(self, runner, paths): + _mock_models() + settings_path, _ = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text(json.dumps({"theme": "dark"})) + assert _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto").exit_code == 0 + edited = json.loads(settings_path.read_text()) + edited["env"] = {key: f"{value}-edited" for key, value in edited["env"].items()} + edited["model"] = "mine" + settings_path.write_text(json.dumps(edited)) + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert "Nothing in" in result.output and "was still ours to restore" in result.output + assert "Left as you changed them since:" in result.output and "model" in result.output + + @responses.activate + def test_names_the_server_a_withheld_credential_was_captured_with_and_keeps_the_receipt(self, runner, paths): + _mock_models() + settings_path, state_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text( + json.dumps({"env": {"ANTHROPIC_BASE_URL": "https://api.anthropic.com", "ANTHROPIC_API_KEY": "sk-ant"}}) + ) + assert _configure(runner, "--api-key", VALID_KEY).exit_code == 0 + edited = json.loads(settings_path.read_text()) + edited["env"]["ANTHROPIC_BASE_URL"] = "http://other-proxy:4000" + settings_path.write_text(json.dumps(edited)) + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert "env.ANTHROPIC_API_KEY (captured with https://api.anthropic.com)" in result.output + assert str(state_path) in result.output and state_path.exists() + assert "sk-ant" not in result.output + + def test_refuses_while_lite_up_holds_a_backup(self, runner, paths, lite_up_backup): + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code != 0 and "lite down" in result.output + + def test_without_a_receipt_it_fails_loudly(self, runner, paths): + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code != 0 + assert "nothing to undo" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_pi.py b/tests/test_litellm/proxy/client/cli/test_pi.py index 68c0ac70064..1c2da514ee2 100644 --- a/tests/test_litellm/proxy/client/cli/test_pi.py +++ b/tests/test_litellm/proxy/client/cli/test_pi.py @@ -4,9 +4,11 @@ import stat from concurrent.futures import ThreadPoolExecutor from pathlib import Path +import pytest import requests from litellm.proxy.client.cli.commands.pi import ( + ListingFailure, ModelLimits, PiSyncError, fetch_model_ids, @@ -28,6 +30,10 @@ class _FakeResponse: return self._payload +def _refused(*args, **kwargs): + raise requests.ConnectionError("refused") + + class TestFetchModelIds: def test_returns_ids_in_proxy_order_deduped(self): captured = {} @@ -53,9 +59,7 @@ class TestFetchModelIds: assert "Could not list models" in result.message def test_non_200_is_a_value(self): - result = fetch_model_ids( - "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500) - ) + result = fetch_model_ids("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500)) assert isinstance(result, PiSyncError) assert "HTTP 500" in result.message @@ -75,6 +79,22 @@ class TestFetchModelIds: ) assert isinstance(result, PiSyncError) assert "no models" in result.message + assert result.kind is ListingFailure.EMPTY + + @pytest.mark.parametrize( + ("get", "kind"), + [ + (_refused, ListingFailure.UNREACHABLE), + (lambda *a, **k: _FakeResponse(401), ListingFailure.REJECTED), + (lambda *a, **k: _FakeResponse(403), ListingFailure.REJECTED), + (lambda *a, **k: _FakeResponse(500), ListingFailure.OTHER), + (lambda *a, **k: _FakeResponse(200), ListingFailure.BAD_BODY), + ], + ids=["unreachable", "401", "403", "500", "bad-body"], + ) + def test_the_failure_kind_is_decided_where_the_response_is_classified(self, get, kind): + result = fetch_model_ids("http://localhost:4000", "sk-key", get=get) + assert isinstance(result, PiSyncError) and result.kind is kind class TestFetchModelLimits: diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index aead1764b0e..b607b1a3db7 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -11,11 +11,11 @@ from click.testing import CliRunner from litellm.proxy.client.cli.commands import up as up_module from litellm.proxy.client.cli.commands.agents import AgentRunError -from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError +from litellm.proxy.client.cli.commands.claude_settings import ApiKeyHelper, ClaudeSettingsError from litellm.proxy.client.cli.commands.up import ( BackupRecord, UpError, - _ensure_fresh_login, + ensure_fresh_login, down, load_json_or_empty, merge_claude_settings, @@ -40,12 +40,12 @@ def _patch_paths(monkeypatch, tmp_path): class TestMergeClaudeSettings: def test_preserves_unrelated_top_level_keys(self): - merged = merge_claude_settings({"theme": "dark"}, "http://localhost:4000", "helper") + merged = merge_claude_settings({"theme": "dark"}, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["theme"] == "dark" def test_preserves_unrelated_env_keys(self): settings = {"env": {"SOME_OTHER_VAR": "value"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["SOME_OTHER_VAR"] == "value" def test_overrides_base_url_and_helper(self): @@ -53,7 +53,7 @@ class TestMergeClaudeSettings: "env": {"ANTHROPIC_BASE_URL": "https://old.example.com"}, "apiKeyHelper": "old-helper", } - merged = merge_claude_settings(settings, "http://localhost:4000/", "new-helper") + merged = merge_claude_settings(settings, "http://localhost:4000/", ApiKeyHelper("new-helper")) assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000" assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" @@ -61,21 +61,21 @@ class TestMergeClaudeSettings: def test_preserves_existing_gateway_model_discovery(self): settings = {"env": {"CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY": "0"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "0" def test_preserves_existing_tool_search(self): settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" def test_drops_stray_api_key(self): settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert "ANTHROPIC_API_KEY" not in merged["env"] def test_works_from_empty_settings(self): - merged = merge_claude_settings({}, "http://localhost:4000", "helper") + merged = merge_claude_settings({}, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"] == { "ANTHROPIC_BASE_URL": "http://localhost:4000", "ENABLE_TOOL_SEARCH": "true", @@ -85,7 +85,7 @@ class TestMergeClaudeSettings: def test_does_not_mutate_input(self): settings = {"env": {"FOO": "bar"}} - merge_claude_settings(settings, "http://localhost:4000", "helper") + merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert settings == {"env": {"FOO": "bar"}} @@ -327,7 +327,7 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [] @@ -339,7 +339,7 @@ class TestEnsureFreshLogin: monkeypatch, on_login=lambda: store.log_in({"key": "sk-b", "base_url": "http://proxy-b:4000"}, "sk-b") ) - _ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) assert login_calls == [("http://proxy-b:4000", False)] assert store.key_requests == ["http://proxy-b:4000", "http://proxy-b:4000"] @@ -353,7 +353,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in({"key": "sk-a", "base_url": "http://proxy-a:4000"}, "sk-a"), ) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [("http://proxy-a:4000", False)] @@ -363,7 +363,7 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) with pytest.raises(UpError, match="Run `lite login` first"): - _ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) def test_trusts_a_pkce_credential_that_was_renewed_on_the_way_in(self, monkeypatch): """A --pkce key inside its freshness buffer is renewed by `get_stored_api_key`, so `lite up` @@ -377,7 +377,7 @@ class TestEnsureFreshLogin: ) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [] assert store.key_requests == ["http://proxy-a:4000"] @@ -390,7 +390,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in(_pkce_record("http://proxy-a:4000", seconds_left=86_400), "sk-pkce-fresh"), ) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [("http://proxy-a:4000", True)] @@ -399,7 +399,7 @@ class TestEnsureFreshLogin: _FakeTokenStore(monkeypatch, _pkce_record("http://proxy-a:4000", seconds_left=-10), {}) with pytest.raises(UpError, match="Run `lite login --pkce` first"): - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) def test_trusts_the_key_the_cli_group_already_resolved_instead_of_reading_the_token_file_again( self, monkeypatch @@ -409,7 +409,7 @@ class TestEnsureFreshLogin: store = _FakeTokenStore(monkeypatch, _pkce_record("http://proxy-a:4000", seconds_left=86_400), {}) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key="sk-pkce-renewed-by-the-group")) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key="sk-pkce-renewed-by-the-group")) assert login_calls == [] assert store.key_requests == [] @@ -423,7 +423,7 @@ class TestEnsureFreshLogin: ) with pytest.raises(UpError, match="Run `lite login --pkce` first"): - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) assert store.key_requests == [] @@ -435,7 +435,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in(_pkce_record("http://proxy-a:4000", seconds_left=86_400), "sk-pkce-fresh"), ) - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) assert login_calls == [("http://proxy-a:4000", True)] assert store.key_requests == ["http://proxy-a:4000"] From e34c4c8edc5fc6cd70fccf6a44c7765d920a6e28 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 18:09:28 -0700 Subject: [PATCH 79/81] fix(ui): scope shadow eval models to configured chat groups (#40488) Co-authored-by: Claude Code --- .../_components/ShadowEvalSection.test.tsx | 151 ++++++++++++++---- .../_components/ShadowEvalStartForm.tsx | 81 ++++------ .../hooks/models/useModels.test.ts | 103 +++++++++--- .../app/(dashboard)/hooks/models/useModels.ts | 61 ++++--- .../components/chat_ui/EndpointUtils.tsx | 35 +--- .../chat_ui/mode_endpoint_mapping.tsx | 17 ++ .../src/lib/autorouter_presets.test.ts | 13 ++ .../src/lib/autorouter_presets.ts | 13 +- 8 files changed, 317 insertions(+), 157 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 64363da9933..1f73671caae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -1,3 +1,4 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; @@ -14,7 +15,9 @@ vi.mock("./useShadowEval", () => ({ })); const authorizedRoleMock = vi.fn(() => ({ accessToken: "token", isViewOnly: false })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => authorizedRoleMock() })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ userId: "test-user-id", userRole: "Admin", ...authorizedRoleMock() }), +})); vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useInfiniteKeys: vi.fn(() => ({ @@ -68,27 +71,33 @@ vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ })), })); -vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ +vi.mock("@/app/(dashboard)/hooks/models/useModels", async (importOriginal) => ({ + ...(await importOriginal()), useAutoRouters: vi.fn(() => ({ data: [ { model_name: "claude-auto", litellm_params: { model: "auto_router/claude-auto" } }, { model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } }, ], })), - usePlainModelGroups: vi.fn(() => new Set(["prod-claude"])), + usePlainModelGroups: vi.fn(() => new Set(["prod-claude", "prod-judge"])), + usePlainChatModelGroups: vi.fn(() => new Set(["prod-claude", "prod-judge"])), + usePlainChatModelDeployments: vi.fn(() => [ + { + model_name: "prod-judge", + litellm_params: { model: "anthropic/claude-sonnet-5" }, + model_info: { mode: "chat" }, + }, + ]), })); -vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ - useModelCostMap: vi.fn(() => ({ - data: { - "claude-sonnet-5": { litellm_provider: "anthropic", mode: "chat" }, - "gpt-4o": { litellm_provider: "openai", mode: "chat" }, - "gemini/gemini-2.5-pro": { litellm_provider: "gemini", mode: "chat" }, - "text-embedding-3-large": { litellm_provider: "openai", mode: "embedding" }, - }, - })), +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + modelInfoCall: vi.fn(), })); +import { usePlainChatModelGroups, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { modelInfoCall } from "@/components/networking"; + import ShadowEvalSection, { shadowedTargetLabel } from "./ShadowEvalSection"; import { useShadowEvalJob, @@ -107,7 +116,7 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ models: [], direction: "forward", baseline_model: null, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", shadow_percentage: 10, targets: [ { @@ -249,6 +258,85 @@ describe("ShadowEvalSection", () => { if (defaultKeysImpl) vi.mocked(useInfiniteKeys).mockImplementation(defaultKeysImpl); }); + it("labels only configured judge recommendations", async () => { + const user = userEvent.setup(); + mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Select a judge model")); + expect(screen.getByRole("option", { name: /prod-judge.*Recommended/ })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); + + await user.keyboard("{Escape}"); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + expect(screen.getByRole("option", { name: "prod-judge", exact: true })).toBeInTheDocument(); + expect(screen.queryByText("Recommended")).not.toBeInTheDocument(); + }); + + it("keeps custom models selectable through the real model hooks without widening chat choices to traffic filters", async () => { + const hooks = await vi.importActual( + "@/app/(dashboard)/hooks/models/useModels", + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const deployments = [ + { model_name: "custom-chat", litellm_params: { model: "openai/private-chat" } }, + { model_name: "custom-judge", litellm_params: { model: "openai/private-judge" }, model_info: { mode: null } }, + { + model_name: "embedding", + litellm_params: { model: "openai/private-embedding" }, + model_info: { mode: "embedding" }, + }, + { + model_name: "responses-only", + litellm_params: { model: "openai/private-responses" }, + model_info: { mode: "responses" }, + }, + { model_name: "auto-router", litellm_params: { model: "auto_router/complexity_router" } }, + ]; + vi.mocked(modelInfoCall).mockResolvedValue({ data: deployments, total_pages: 1 }); + const user = userEvent.setup(); + const { start } = mockHooks({}); + await vi.mocked(usePlainModelGroups).withImplementation(hooks.usePlainModelGroups, async () => { + await vi.mocked(usePlainChatModelGroups).withImplementation(hooks.usePlainChatModelGroups, async () => { + render( + + + , + ); + await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "responses-only"); + await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "custom-chat"); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); + await user.click(screen.getByPlaceholderText("Search keys by alias")); + await user.click(within(await screen.findByTestId("paginated-multi-select-list")).getByText("prod-alpha")); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); + await user.click(screen.getByPlaceholderText("Select a judge model")); + expect(screen.getAllByRole("option")).toHaveLength(2); + expect(screen.getByRole("option", { name: "custom-chat", exact: true })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "custom-judge", exact: true })).toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "custom-judge", exact: true })); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + expect(screen.getAllByRole("option")).toHaveLength(2); + expect(screen.getByRole("option", { name: "custom-chat", exact: true })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "custom-judge", exact: true })).toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "custom-chat", exact: true })); + await user.click(screen.getByText("Start shadow eval")); + expect(start.mutate).toHaveBeenCalledWith( + expect.objectContaining({ judge_model: "custom-judge", baseline_model: "custom-chat", models: [] }), + ); + }); + }); + client.clear(); + }); + it("offers the start form while the list is still loading", () => { mockHooks({ isPending: true }); render(); @@ -444,7 +532,8 @@ describe("ShadowEvalSection", () => { expect(screen.getByText("Start shadow eval")).toBeDisabled(); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -457,7 +546,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -474,7 +563,7 @@ describe("ShadowEvalSection", () => { await user.click(within(teamList).getByText("engineering")); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -487,7 +576,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -503,7 +592,7 @@ describe("ShadowEvalSection", () => { await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "prod-claude"); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); expect(start.mutate).toHaveBeenCalledWith( @@ -524,20 +613,23 @@ describe("ShadowEvalSection", () => { expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument(); expect(screen.getByPlaceholderText("Every model the targets use")).toBeInTheDocument(); - await user.click(screen.getByText("Adoption check: key's traffic vs the router")); - await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); expect(screen.queryByPlaceholderText("Every model the targets use")).not.toBeInTheDocument(); await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); expect(screen.getByText("Start shadow eval")).toBeDisabled(); await user.click(screen.getByPlaceholderText("Select a baseline model")); - expect(await screen.findByRole("option", { name: /openai\/gpt-4o/ })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("option", { name: /prod-claude/ })); await user.click(screen.getByText("Start shadow eval")); @@ -552,7 +644,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -574,7 +666,7 @@ describe("ShadowEvalSection", () => { screen.getByText("Every router sees the same sampled requests, judged against the same live responses"), ).toBeInTheDocument(); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -587,7 +679,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -605,10 +697,13 @@ describe("ShadowEvalSection", () => { await user.click(await screen.findByText("gpt-auto")); await user.click(routerInput); await user.click(await screen.findByText("claude-auto")); - await user.click(screen.getByText("Adoption check: key's traffic vs the router")); - await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByPlaceholderText("Select a baseline model")); await user.click(screen.getByRole("option", { name: /prod-claude/ })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx index 2eb5fa9c945..d85d26a21a8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -5,8 +5,13 @@ import React, { useMemo, useState } from "react"; import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; -import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { + useAutoRouters, + usePlainChatModelDeployments, + usePlainChatModelGroups, + usePlainModelGroups, +} from "@/app/(dashboard)/hooks/models/useModels"; +import { buildModelAvailability, deploymentRefsFromModelInfo, resolveAvailableModels } from "@/lib/autorouter_presets"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; import TeamMultiSelect from "@/components/common_components/team_multi_select"; @@ -24,53 +29,8 @@ type ShadowEvalDirection = ShadowEvalJob["direction"]; const MAX_ROUTERS = 4; const MAX_MODELS = 100; - const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const; -interface CostMapEntry { - litellm_provider?: string; - mode?: string; -} - -const useChatModelNames = (): string[] => { - const { data: costMap } = useModelCostMap(); - return useMemo(() => { - if (!costMap) return []; - const chatModels = Object.entries(costMap as Record) - .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) - .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); - return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); - }, [costMap]); -}; - -const useJudgeModelOptions = (): SearchSelectOption[] => { - const chatModels = useChatModelNames(); - return useMemo(() => { - const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ - label: model, - value: model, - sublabel: "Recommended", - })); - const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); - const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); - return [...pinned, ...rest]; - }, [chatModels]); -}; - -const useBaselineModelOptions = (): SearchSelectOption[] => { - const configuredGroups = usePlainModelGroups(); - const chatModels = useChatModelNames(); - return useMemo(() => { - const configured = [...configuredGroups] - .toSorted((a, b) => a.localeCompare(b)) - .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); - const rest = chatModels - .filter((model) => !configuredGroups.has(model)) - .map((model) => ({ label: model, value: model })); - return [...configured, ...rest]; - }, [configuredGroups, chatModels]); -}; - const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ { value: "forward", label: "Adoption check: key's traffic vs the router" }, { value: "reverse", label: "Regression check: router's picks vs a baseline" }, @@ -276,13 +236,32 @@ export const StartForm: React.FC = () => { const [judgeModel, setJudgeModel] = useState(""); const [maxBudget, setMaxBudget] = useState("10"); const { data: autoRouters } = useAutoRouters(); - const judgeModelOptions = useJudgeModelOptions(); - const baselineModelOptions = useBaselineModelOptions(); const configuredGroups = usePlainModelGroups(); + const chatGroups = usePlainChatModelGroups(); + const chatDeployments = usePlainChatModelDeployments(); const modelOptions = useMemo( () => [...configuredGroups].toSorted((a, b) => a.localeCompare(b)).map((name) => ({ label: name, value: name })), [configuredGroups], ); + const chatOptions = useMemo( + () => modelOptions.filter((option) => chatGroups.has(option.value)), + [modelOptions, chatGroups], + ); + const chatAvailability = useMemo( + () => buildModelAvailability(chatGroups, deploymentRefsFromModelInfo(chatDeployments)), + [chatDeployments, chatGroups], + ); + const recommendedJudgeModels = useMemo( + () => new Set(RECOMMENDED_JUDGE_MODELS.flatMap((model) => resolveAvailableModels(model, chatAvailability))), + [chatAvailability], + ); + const judgeOptions = useMemo( + () => + chatOptions.map((option) => + recommendedJudgeModels.has(option.value) ? { ...option, sublabel: "Recommended" } : option, + ), + [chatOptions, recommendedJudgeModels], + ); const start = useStartShadowEval(); const routerOptions = useMemo(() => { @@ -434,7 +413,7 @@ export const StartForm: React.FC = () => { {direction === "reverse" && ( { )} { }); }); -describe("selectPlainModelGroups", () => { - it("keeps only non-auto-router model groups", () => { - const deployments: AutoRouterCandidateDeployment[] = [ - { model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } }, - { model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } }, - { model_name: "claude-sonnet", litellm_params: { model: "anthropic/claude-sonnet-4-5" } }, - { model_name: "cheap-router", litellm_params: { model: "auto_router/adaptive_router" } }, +describe("selectPlainChatModelGroups", () => { + it("keeps chat-capable groups when mode metadata is absent or any sibling is compatible", () => { + const deployments: AutoRouterDeployment[] = [ + { model_name: "no-info" }, + { model_name: "null-info", model_info: null }, + { model_name: "empty-info", model_info: {} }, + { model_name: "missing-mode", model_info: { db_model: false } }, + { model_name: "null-mode", model_info: { mode: null } }, + { model_name: "empty-mode", model_info: { mode: "" } }, + { model_name: "chat", model_info: { mode: "chat", db_model: true } }, + { model_name: "completion", model_info: { mode: "completion" } }, + { model_name: "chat-and-missing", model_info: { mode: "chat" } }, + { model_name: "chat-and-missing" }, + { model_name: "chat-then-embedding", model_info: { mode: "chat" } }, + { model_name: "chat-then-embedding", model_info: { mode: "embedding" } }, + { model_name: "embedding-then-chat", model_info: { mode: "embedding" } }, + { model_name: "embedding-then-chat", model_info: { mode: "chat" } }, + { model_name: "embedding-only", model_info: { mode: "embedding" } }, + { model_name: "speech-only", model_info: { mode: "speech" } }, + { model_name: "shared-router", litellm_params: { model: "openai/gpt-4o" } }, + { model_name: "shared-router", litellm_params: { model: "auto_router/complexity_router" } }, + { model_name: "", model_info: { mode: "chat" } }, ]; - expect(selectPlainModelGroups(deployments)).toEqual(new Set(["claude-haiku", "claude-sonnet"])); - }); - - it("drops a group name that also fronts an auto-router deployment", () => { - const deployments: AutoRouterCandidateDeployment[] = [ - { model_name: "shared-name", litellm_params: { model: "auto_router/complexity_router" } }, - { model_name: "shared-name", litellm_params: { model: "anthropic/claude-sonnet-4-5" } }, - ]; - - expect(selectPlainModelGroups(deployments)).toEqual(new Set()); - }); - - it("drops deployments that have no public model_name", () => { - expect(selectPlainModelGroups([{ model_name: "", litellm_params: { model: "openai/gpt-4o" } }])).toEqual(new Set()); + expect(selectPlainChatModelGroups(deployments)).toEqual( + new Set([ + "no-info", + "null-info", + "empty-info", + "missing-mode", + "null-mode", + "empty-mode", + "chat", + "completion", + "chat-and-missing", + "chat-then-embedding", + "embedding-then-chat", + ]), + ); }); }); @@ -1103,6 +1121,47 @@ describe("useAutoRouterModelGroups", () => { expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 3, 1000); }); + it("uses every page for configured chat groups and keeps custom deployments without mode metadata", async () => { + (modelInfoCall as any).mockImplementation((_t: string, _u: string, _r: string, page: number) => + Promise.resolve( + page === 1 + ? { + data: [ + { model_name: "configured-chat", model_info: { mode: "chat" } }, + { model_name: "embedding-only", model_info: { mode: "embedding" } }, + ], + total_pages: 2, + } + : { + data: [ + { model_name: "custom-no-mode", model_info: { db_model: true } }, + { model_name: "speech-only", model_info: { mode: "speech" } }, + ], + total_pages: 2, + }, + ), + ); + + const { result } = renderHook(() => usePlainChatModelGroups(), { wrapper }); + + await waitFor(() => expect(result.current.size).toBe(2)); + expect(result.current).toEqual(new Set(["configured-chat", "custom-no-mode"])); + expect(modelInfoCall).toHaveBeenCalledTimes(2); + }); + + it("returns an empty chat group set while loading and after failure", async () => { + (modelInfoCall as any).mockReturnValueOnce(new Promise(() => {})); + const loading = renderHook(() => usePlainChatModelGroups(), { wrapper }); + expect(loading.result.current).toEqual(new Set()); + loading.unmount(); + + queryClient.clear(); + (modelInfoCall as any).mockRejectedValueOnce(new Error("boom")); + const failed = renderHook(() => usePlainChatModelGroups(), { wrapper }); + await waitFor(() => expect(modelInfoCall).toHaveBeenCalledTimes(2)); + expect(failed.result.current).toEqual(new Set()); + }); + it("returns an empty set before the model list resolves", () => { (modelInfoCall as any).mockReturnValue(new Promise(() => {})); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index b3a783a71dc..579ee7ff81a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -2,6 +2,7 @@ import { useQuery, useInfiniteQuery, useQueryClient, UseQueryResult } from "@tan import { createQueryKeys } from "../common/queryKeysFactory"; import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "../useAuthorized"; +import { EndpointType, isModeCompatibleWithEndpoint } from "@/components/chat_ui/mode_endpoint_mapping"; export interface ProxyModel { id: string; @@ -87,6 +88,7 @@ export const useModelsInfo = ( const AUTO_ROUTER_MODEL_PREFIX = "auto_router/"; const AUTO_ROUTER_LOOKUP_PAGE_SIZE = 1000; const NO_AUTO_ROUTERS: ReadonlySet = new Set(); +const NO_DEPLOYMENTS: AutoRouterDeployment[] = []; export interface AutoRouterCandidateDeployment { model_name?: string | null; @@ -96,6 +98,7 @@ export interface AutoRouterCandidateDeployment { export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { litellm_params?: { model?: string | null; + base_model?: string | null; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; @@ -111,6 +114,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + mode?: string | null; created_at?: string | null; updated_at?: string | null; team_id?: string | null; @@ -142,6 +146,22 @@ export const selectPlainModelGroups = (deployments: AutoRouterCandidateDeploymen ); }; +export const selectPlainChatModelDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] => { + const plainGroups = selectPlainModelGroups(deployments); + return deployments.filter( + (deployment) => + plainGroups.has(deployment.model_name ?? "") && + isModeCompatibleWithEndpoint(deployment.model_info?.mode, EndpointType.CHAT), + ); +}; + +export const selectPlainChatModelGroups = (deployments: AutoRouterDeployment[]): ReadonlySet => + new Set( + selectPlainChatModelDeployments(deployments) + .map((deployment) => deployment.model_name) + .filter((name): name is string => Boolean(name)), + ); + export const fetchAllModelDeployments = async ( accessToken: string, userId: string, @@ -180,37 +200,32 @@ export const autoRouterListKey = (userId: string | null, userRole: string | null }, }); -export const useAutoRouterModelGroups = (): ReadonlySet => { +const useDeployments = ( + select: (deployments: AutoRouterDeployment[]) => TSelected, +): UseQueryResult => { const { accessToken, userId, userRole } = useAuthorized(); - const { data } = useQuery>({ + return useQuery({ queryKey: autoRouterListKey(userId, userRole), queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), enabled: Boolean(accessToken && userId && userRole), - select: selectAutoRouterModelGroups, + select, }); - return data ?? NO_AUTO_ROUTERS; }; -export const usePlainModelGroups = (): ReadonlySet => { - const { accessToken, userId, userRole } = useAuthorized(); - const { data } = useQuery>({ - queryKey: autoRouterListKey(userId, userRole), - queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), - enabled: Boolean(accessToken && userId && userRole), - select: selectPlainModelGroups, - }); - return data ?? NO_AUTO_ROUTERS; -}; +export const useAutoRouterModelGroups = (): ReadonlySet => + useDeployments(selectAutoRouterModelGroups).data ?? NO_AUTO_ROUTERS; -export const useAutoRouters = (): UseQueryResult => { - const { accessToken, userId, userRole } = useAuthorized(); - return useQuery({ - queryKey: autoRouterListKey(userId, userRole), - queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), - enabled: Boolean(accessToken && userId && userRole), - select: selectAutoRouterDeployments, - }); -}; +export const usePlainModelGroups = (): ReadonlySet => + useDeployments(selectPlainModelGroups).data ?? NO_AUTO_ROUTERS; + +export const usePlainChatModelGroups = (): ReadonlySet => + useDeployments(selectPlainChatModelGroups).data ?? NO_AUTO_ROUTERS; + +export const usePlainChatModelDeployments = (): AutoRouterDeployment[] => + useDeployments(selectPlainChatModelDeployments).data ?? NO_DEPLOYMENTS; + +export const useAutoRouters = (): UseQueryResult => + useDeployments(selectAutoRouterDeployments); export const useInvalidateAutoRouters = (): (() => Promise) => { const queryClient = useQueryClient(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx index 8fd4a57dbfd..87d569f6490 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx @@ -1,7 +1,9 @@ import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { EndpointType, getEndpointType, ModelMode } from "@/components/chat_ui/mode_endpoint_mapping"; - -const KNOWN_MODEL_MODES = new Set(Object.values(ModelMode)); +import { + EndpointType, + getEndpointType, + isModeCompatibleWithEndpoint, +} from "@/components/chat_ui/mode_endpoint_mapping"; export const determineEndpointType = (selectedModel: string, modelInfo: ModelGroup[]): EndpointType => { const selectedModelInfo = modelInfo.find((option) => option.model_group === selectedModel); @@ -13,31 +15,8 @@ export const determineEndpointType = (selectedModel: string, modelInfo: ModelGro return EndpointType.CHAT; }; -export const isModelCompatibleWithEndpoint = (model: ModelGroup, endpointType: EndpointType): boolean => { - if (!model.mode) { - return true; - } - - if (!KNOWN_MODEL_MODES.has(model.mode)) { - return false; - } - - const optionEndpoint = getEndpointType(model.mode); - - if ( - endpointType === EndpointType.RESPONSES || - endpointType === EndpointType.ANTHROPIC_MESSAGES || - endpointType === EndpointType.INTERACTIONS - ) { - return optionEndpoint === endpointType || optionEndpoint === EndpointType.CHAT; - } - - if (endpointType === EndpointType.IMAGE_EDITS) { - return optionEndpoint === endpointType || optionEndpoint === EndpointType.IMAGE; - } - - return optionEndpoint === endpointType; -}; +export const isModelCompatibleWithEndpoint = (model: ModelGroup, endpointType: EndpointType): boolean => + isModeCompatibleWithEndpoint(model.mode, endpointType); export const filterModelsForEndpoint = (models: ModelGroup[], endpointType: EndpointType): ModelGroup[] => models.filter((model) => isModelCompatibleWithEndpoint(model, endpointType)); diff --git a/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx index 930ded5d1a5..b46200ccbf2 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx @@ -57,3 +57,20 @@ export const getEndpointType = (mode: string): EndpointType => { // else default to chat return EndpointType.CHAT; }; + +export const isModeCompatibleWithEndpoint = (mode: string | null | undefined, endpointType: EndpointType): boolean => { + if (!mode) return true; + if (!Object.values(ModelMode).includes(mode as ModelMode)) return false; + const optionEndpoint = getEndpointType(mode); + if ( + endpointType === EndpointType.RESPONSES || + endpointType === EndpointType.ANTHROPIC_MESSAGES || + endpointType === EndpointType.INTERACTIONS + ) { + return optionEndpoint === endpointType || optionEndpoint === EndpointType.CHAT; + } + if (endpointType === EndpointType.IMAGE_EDITS) { + return optionEndpoint === endpointType || optionEndpoint === EndpointType.IMAGE; + } + return optionEndpoint === endpointType; +}; diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 8a5f83adbdf..fed11454c23 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -13,6 +13,7 @@ import { buildModelAvailability, deploymentRefsFromModelInfo, normalizeModelName, + resolveAvailableModels, } from "./autorouter_presets"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; @@ -380,6 +381,18 @@ describe("autorouter_presets", () => { expect(availability.underlyingIndex.size).toBe(0); }); + it("returns every configured group serving the same underlying model", () => { + const availability = buildModelAvailability( + ["z-group", "a-group"], + [ + { modelGroup: "z-group", underlyingModels: ["anthropic/claude-sonnet-5"] }, + { modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-sonnet-5-v1:0"] }, + ], + ); + + expect(resolveAvailableModels("anthropic/claude-sonnet-5", availability)).toEqual(["a-group", "z-group"]); + }); + it("breaks ties between groups serving the same model deterministically, alphabetically", () => { const availability = buildModelAvailability( ["z-group", "a-group"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index f2df55fc310..02096cada41 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -159,16 +159,19 @@ export const deploymentRefsFromModelInfo = ( return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : []; }); -export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { +export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => { const { modelGroups, underlyingIndex } = availability; - if (modelGroups.has(requiredModel)) return requiredModel; + if (modelGroups.has(requiredModel)) return [requiredModel]; const normalized = normalizeModelName(requiredModel); - const groupMatch = Array.from(modelGroups).find((available) => normalizeModelName(available) === normalized); - if (groupMatch !== undefined) return groupMatch; + const groupMatches = Array.from(modelGroups).filter((available) => normalizeModelName(available) === normalized); + if (groupMatches.length > 0) return groupMatches; const key = normalizeUnderlyingModel(requiredModel); - return key === null ? undefined : underlyingIndex.get(key)?.[0]; + return key === null ? [] : underlyingIndex.get(key) ?? []; }; +export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => + resolveAvailableModels(requiredModel, availability)[0]; + export const getMissingModels = ( config: Parameters[0], availability: ModelAvailability, From 3aa9267df278a4a28399460cc7cbe4708a4bd169 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 18:19:52 -0700 Subject: [PATCH 80/81] fix(ui): drop blank streamed costs instead of reading them as zero Number("") and Number(" ") both return 0, which passed the finite check, so a provider reporting an empty cost got a fabricated $0.000000 metric instead of having the unusable value omitted. Both ingestion sites carried the same inline parsing, so this pulls it into one parseUsageCost helper that keeps finite numbers and non-blank numeric strings and drops everything else, including booleans, arrays and breakdown objects. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YNw8WvkvCSeTcE5qvergu3 --- .../llm_calls/chat_completion.test.tsx | 6 +++ .../components/llm_calls/chat_completion.tsx | 10 ++--- .../llm_calls/responses_api.test.tsx | 29 ++++++++++++++ .../components/llm_calls/responses_api.tsx | 9 ++--- .../components/llm_calls/usage_cost.test.ts | 39 +++++++++++++++++++ .../src/components/llm_calls/usage_cost.ts | 23 +++++++++++ 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts create mode 100644 ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx index 129d51d50a2..71f36bdb6db 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx @@ -472,6 +472,12 @@ describe("chat_completion prompt cache usage", () => { expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); }); + + it("omits cost when the provider reports a blank value", async () => { + const usageData = await captureUsage({ cost: " " }); + + expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); + }); }); describe("chat_completion response cache", () => { diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index c12a2c3cfb4..ffa2877fbd9 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -5,6 +5,7 @@ import { VectorStoreSearchResponse } from "../chat_ui/types"; import { getProxyBaseUrl } from "@/components/networking"; import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import { parseUsageCost } from "./usage_cost"; const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk => ({ @@ -243,12 +244,9 @@ export async function makeOpenAIChatCompletionRequest( usageData.reasoningTokens = chunkWithUsage.usage.completion_tokens_details.reasoning_tokens; } - // Extract cost from usage object if available - if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - const parsedCost = Number(chunkWithUsage.usage.cost); - if (Number.isFinite(parsedCost)) { - usageData.cost = parsedCost; - } + const parsedCost = parseUsageCost(chunkWithUsage.usage.cost); + if (parsedCost !== undefined) { + usageData.cost = parsedCost; } onUsageData(usageData); diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 7b13f0ac2e3..b55df89d5cc 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -260,6 +260,35 @@ describe("responses_api", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); }); + it("should omit cost when the proxy reports a blank cost", async () => { + async function* streamWithBlankCost() { + yield { + type: "response.completed", + response: { + id: "resp_blank_cost", + usage: { output_tokens: 12, input_tokens: 12, total_tokens: 24, cost: " " }, + }, + }; + } + mockResponsesCreate.mockResolvedValueOnce(streamWithBlankCost()); + + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, + undefined, + undefined, + undefined, + onUsageData, + ); + + expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); + }); + it("should replay MCP output items as events for a non-streaming response", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 63085a81b94..ce54c7c6b40 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -4,6 +4,7 @@ import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getProxyBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import { parseUsageCost } from "./usage_cost"; import type { MCPEvent } from "@/components/mcp_tools/types"; import { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; import { @@ -311,11 +312,9 @@ export async function makeOpenAIResponsesRequest( usageData.reasoningTokens = reasoningTokens; } - if (usage.cost !== undefined && usage.cost !== null) { - const parsedCost = Number(usage.cost); - if (Number.isFinite(parsedCost)) { - usageData.cost = parsedCost; - } + const parsedCost = parseUsageCost(usage.cost); + if (parsedCost !== undefined) { + usageData.cost = parsedCost; } onUsageData(usageData, mcpToolUsed); diff --git a/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts new file mode 100644 index 00000000000..ee1021c0629 --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { parseUsageCost } from "./usage_cost"; + +describe("parseUsageCost", () => { + it("keeps finite numbers, including zero", () => { + expect(parseUsageCost(0)).toBe(0); + expect(parseUsageCost(0.000063)).toBe(0.000063); + }); + + it("keeps numeric strings", () => { + expect(parseUsageCost("0.00019")).toBe(0.00019); + expect(parseUsageCost(" 0.00019 ")).toBe(0.00019); + }); + + it("drops blank strings instead of fabricating a zero cost", () => { + expect(parseUsageCost("")).toBeUndefined(); + expect(parseUsageCost(" ")).toBeUndefined(); + expect(parseUsageCost("\t\n")).toBeUndefined(); + }); + + it("drops strings with a numeric prefix instead of truncating them", () => { + expect(parseUsageCost("1oops")).toBeUndefined(); + expect(parseUsageCost("0.5 USD")).toBeUndefined(); + }); + + it("drops non-finite numbers", () => { + expect(parseUsageCost(Number.NaN)).toBeUndefined(); + expect(parseUsageCost(Number.POSITIVE_INFINITY)).toBeUndefined(); + }); + + it("drops values that are not numbers or strings", () => { + expect(parseUsageCost(null)).toBeUndefined(); + expect(parseUsageCost(undefined)).toBeUndefined(); + expect(parseUsageCost(true)).toBeUndefined(); + expect(parseUsageCost([])).toBeUndefined(); + expect(parseUsageCost(["0.5"])).toBeUndefined(); + expect(parseUsageCost({ total_cost: 0.5 })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts new file mode 100644 index 00000000000..79f56dcb2df --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts @@ -0,0 +1,23 @@ +/** + * Providers and upstream gateways report `usage.cost` unvalidated: it arrives as a number, a numeric + * string, an empty string, or something non-numeric. A cost that does not resolve to a finite number + * must be dropped rather than coerced, because `NaN` survives `JSON.stringify` as `null` and crashes + * the metrics row on the next load. + */ +export function parseUsageCost(rawCost: unknown): number | undefined { + if (typeof rawCost === "number") { + return Number.isFinite(rawCost) ? rawCost : undefined; + } + + if (typeof rawCost !== "string") { + return undefined; + } + + const trimmed = rawCost.trim(); + if (trimmed === "") { + return undefined; + } + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : undefined; +} From 4adf99557f901fd73525697fdba8c9b2c99d0cc9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:27:12 -0700 Subject: [PATCH 81/81] fix(responses): echo "auto" for a tool_choice the bridge cannot express instead of failing after the provider call --- .../transformation.py | 6 +++-- .../test_litellm_completion_responses.py | 26 +++++++++++++++++++ .../test_streaming_iterator_transformation.py | 20 ++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6b9931b7df2..fca5b0d11cf 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -287,8 +287,10 @@ class LiteLLMCompletionResponsesConfig: return ToolChoiceCustomParam(type="custom", name=custom_name) case _, {"type": "function", "function": {"name": str(function_name)}}: return ToolChoiceFunctionParam(type="function", name=function_name) - case _, normalized: - return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) + case _, "none" | "auto" | "required" as normalized: + return normalized + case _, _: + return "auto" @staticmethod def _should_drop_derived_web_search_options(model: str, custom_llm_provider: str | None) -> bool: diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 8f2937cb252..46249e50572 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1435,6 +1435,9 @@ class TestToolChoiceTransformation: ("required", "required"), ("none", "none"), (None, "auto"), + ("any", "auto"), + ("run_command", "auto"), + ({"name": "run_command"}, "auto"), ], ) def test_transform_tool_choice_for_responses_api_response( @@ -1478,6 +1481,29 @@ class TestToolChoiceTransformation: assert responses_api_response.tool_choice == {"type": "function", "name": "run_command"} + def test_non_streamed_response_with_unrecognized_tool_choice_echoes_auto(self) -> None: + chat_completion_response: Final = ModelResponse( + id="chatcmpl-unrecognized-tool-choice", + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(role="assistant", content="/Users/dev"), + ) + ], + ) + + responses_api_response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Run the command pwd.", + responses_api_request={"tool_choice": "any"}, + chat_completion_response=chat_completion_response, + ) + + assert responses_api_response.tool_choice == "auto" + class TestContentTypeTransformation: """Test content type transformation from Responses API to Chat Completion format""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 70f0957890d..850ee7ba623 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -685,3 +685,23 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: {"type": "function", "name": "run_command"}, ] assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) + + +def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None: + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), + request_input="Run the command pwd.", + responses_api_request={ + "tools": [{"type": "function", "name": "run_command", "parameters": {"type": "object"}}], + "tool_choice": "any", + }, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + + response_events: Final = [ + event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES + ] + + assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"]