From 9082f8e5d0549d93c27009b6d4f6a0a631a73435 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:37:12 +0000 Subject: [PATCH 01/17] feat(bedrock): add Claude Opus 5.5 pricing and capabilities (#42588) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 345 ++++++++++++++++++ model_prices_and_context_window.json | 345 ++++++++++++++++++ 2 files changed, 690 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b48bdd6322c..34cbea0e314 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1773,6 +1773,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1811,6 +1851,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "global.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1849,6 +1929,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "us.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1886,6 +2006,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "eu.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1923,6 +2083,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "au.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1960,6 +2160,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "jp.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -44379,6 +44619,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "us-gov.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -60807,6 +61082,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -61013,6 +61323,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b48bdd6322c..34cbea0e314 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1773,6 +1773,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1811,6 +1851,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "global.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1849,6 +1929,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "us.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1886,6 +2006,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "eu.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1923,6 +2083,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "au.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1960,6 +2160,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "jp.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -44379,6 +44619,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "us-gov.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -60807,6 +61082,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -61013,6 +61323,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, From f7ae9efad2a6db8d588a201f1d644d706944580e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:47:14 -0700 Subject: [PATCH 02/17] test(e2e): hold every worker under an idle RSS budget before any traffic (#42552) * test(e2e): hold every worker under an idle RSS budget before any traffic The harness reads /debug/memory/summary on every replica once at collection time, right after the readiness gate and before this pytest process sends any traffic, and the memory suite's first test fails when any worker idles past E2E_MEMORY_IDLE_RSS_BUDGET_MB (768 MB by default) or gives no reading at all. A v1.100.x worker with a database idled at 836-886 MB where v1.101.0rc1 idled at 544 MB on the same database: prisma-client-py's default recursive type depth generated 91k TypedDict classes that v1.101.0's recursive_type_depth = -1 cut to 19k. The budget starts at the rc1 reading plus headroom. * test(e2e): read idle RSS only when the idle budget test is selected Gate the collection-time /debug/memory/summary read on a selected test using the idle_rss fixture and skip it under --collect-only, so sessions that never run the idle budget test pay no round trip. Drop the markerless unit test file the e2e guide bans and assert live that every configured replica was measured * test(e2e): take the idle RSS read after collection settles Read every replica's RSS from a tryfirst pytest_collection_finish hook so -k and -m deselection has already run, and only when a selected test still asks for the idle_rss fixture and the run is not --collect-only * test(e2e): record the heaviest idle RSS reading as junit properties * test(e2e): attach the idle RSS properties from the harness's setup hook --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/AGENTS.md | 4 +- tests/e2e/conftest.py | 41 +++++++++- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/e2e_config.py | 1 + tests/e2e/memory_readings.py | 78 +++++++++++++++++++ tests/e2e/proxy_client.py | 5 +- .../e2e/router/test_reliability_memory_e2e.py | 63 ++++++++++----- 7 files changed, 167 insertions(+), 26 deletions(-) create mode 100644 tests/e2e/memory_readings.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index ad1e0322787..6212a172fe3 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -43,7 +43,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions` in `test_mcp_chat_completion_oauth_e2e.py` and direct MCP protocol operations in `test_mcp_oauth_happy_path_e2e.py`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion or protocol call lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) +- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests @@ -191,7 +191,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput | session_anomaly | memory (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly | memory | idle_memory (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a2398ef7c3c..f1f94f2d2b3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,6 +46,7 @@ from fixture_mode import pytest_fixture_setup as pytest_fixture_setup from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from memory_readings import RssCapture, read_rss_everywhere from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error @@ -53,6 +54,9 @@ from proxy_client import ProxyClient, build_proxy_client _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +_IDLE_RSS = pytest.StashKey[RssCapture]() + +IDLE_RSS_READ_TIMEOUT_SECONDS: Final = 10.0 OPT_IN_MARKERS: Final = MappingProxyType( { @@ -186,6 +190,16 @@ def _needs_unset_opt_in(item: pytest.Item) -> bool: ) +def _reaches_proxy(item: pytest.Item) -> bool: + """True for a live test that talks to the shared proxy: `e2e`-marked and not a + `migration_startup` test, which boots its own container instead.""" + return item.get_closest_marker("e2e") is not None and item.get_closest_marker("migration_startup") is None + + +def _uses_idle_rss(item: pytest.Item) -> bool: + return isinstance(item, pytest.Function) and "idle_rss" in item.fixturenames + + def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Deselect every test behind an opt-in marker whose env var is unset (see OPT_IN_MARKERS): those tests need a proxy configured differently from the @@ -215,6 +229,20 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item items.sort(key=lambda item: item.get_closest_marker("load") is not None) +@pytest.hookimpl(tryfirst=True) +def pytest_collection_finish(session: pytest.Session) -> None: + """When a selected test asks for the `idle_rss` fixture and this is not a + `--collect-only` run, read every replica's RSS once, right here at the end of + collection and before this process sends any traffic. tryfirst keeps the read + ahead of xdist's own collection-finish report, and the controller schedules no + test until every worker has reported, so this is the idle footprint of a stack + that just passed its readiness gate. The fixture hands the capture to the + idle-budget test in router/test_reliability_memory_e2e.py.""" + if session.config.getoption("collectonly") or not any(_uses_idle_rss(item) for item in session.items): + return + session.config.stash[_IDLE_RSS] = read_rss_everywhere(build_proxy_client(), timeout=IDLE_RSS_READ_TIMEOUT_SECONDS) + + def _liveness_reason(label: str, base_url: str) -> str | None: """None if `base_url` answers its liveness probe, else a failure reason.""" try: @@ -246,7 +274,9 @@ def pytest_runtest_setup(item: pytest.Item) -> None: run even when none is up. Never skip for a missing proxy. Replay mode needs the proxy too: only provider-bound traffic replays from the bundle.""" LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None) - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if _uses_idle_rss(item): + item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties) + if not _reaches_proxy(item): return if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames: return @@ -261,7 +291,7 @@ def pytest_runtest_call(item: pytest.Item) -> None: guard before truncating the spend-log DB. Tests under `tests/e2e/` without the `e2e` marker (pure unit coverage for the harness itself) never hit the proxy, so they must not arm the destructive DB truncate.""" - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if not _reaches_proxy(item): return item.session.stash[_E2E_TEST_RAN] = True @@ -325,6 +355,13 @@ def proxy() -> ProxyClient: return build_proxy_client() +@pytest.fixture(scope="session") +def idle_rss(request: pytest.FixtureRequest) -> RssCapture: + """Every replica's RSS as read once at the end of collection, before this process + sent any traffic (see pytest_collection_finish).""" + return request.config.stash[_IDLE_RSS] + + @pytest.fixture def resources(client: ProxyClientProvider) -> Iterator[ResourceManager]: """init -> run -> teardown: create a manager, run the test, release resources. diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 9f88f2478c0..5040f5f4dcf 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -38,4 +38,5 @@ - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} - {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"} +- {id: reliability.perf.idle_memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: idle_memory, assertions: [under_slo], exercised_on: [], source: grammar, rationale: "Every worker's RSS as read right after the readiness gate and before any test traffic stays under a fixed idle budget; a DB-backed v1.100.x worker idled at 886 MB against a 2 GiB pod limit where v1.101.0rc1 idled at 544 MB"} - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 77395a066ac..ac26b2a3875 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -173,6 +173,7 @@ MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4")) MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15")) MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48")) +MEMORY_IDLE_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_IDLE_RSS_BUDGET_MB", "768")) MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) diff --git a/tests/e2e/memory_readings.py b/tests/e2e/memory_readings.py new file mode 100644 index 00000000000..37a27c455f8 --- /dev/null +++ b/tests/e2e/memory_readings.py @@ -0,0 +1,78 @@ +"""Per-worker RSS readings of the proxy through /debug/memory/summary. + +One read goes to every configured replica (PROXY_REPLICA_URLS) under the master +key and answers from whichever worker behind that address took the connection; +the release stack runs one worker per gateway replica, so a read per replica is +a read per worker. A reading keys its worker by replica address, hostname, and +pid, since pods in their own pid namespaces report the same pids. A replica that +gives no reading (unreachable, a non-2xx, or a summary without ram_usage_mb) is +kept as a failure reason rather than dropped, so a test can fail on it by name +instead of passing on the replicas that did answer. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from e2e_http import Result, Success +from models import MemorySummaryResponse +from proxy_client import ProxyClient + +WorkerKey = tuple[str, str | None, int] + + +@dataclass(frozen=True, slots=True) +class RssReading: + replica: str + hostname: str | None + worker_pid: int + ram_usage_mb: float + + @property + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) + + @property + def where(self) -> str: + return f"worker pid {self.worker_pid} on {self.hostname or 'an unnamed host'} behind {self.replica}" + + +@dataclass(frozen=True, slots=True) +class RssCapture: + readings: tuple[RssReading, ...] + failures: tuple[str, ...] + + @property + def heaviest(self) -> RssReading | None: + return max(self.readings, key=lambda reading: reading.ram_usage_mb, default=None) + + @property + def junit_properties(self) -> tuple[tuple[str, object], ...]: + heaviest: Final = self.heaviest + if heaviest is None: + return () + return (("idle_rss_heaviest_mb", heaviest.ram_usage_mb), ("idle_rss_heaviest_worker", heaviest.where)) + + +def _outcome(replica: str, result: Result[MemorySummaryResponse]) -> RssReading | str: + match result: + case Success(data=body) if body.memory.ram_usage_mb is not None: + return RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) + case Success(data=body): + return f"{replica} answered /debug/memory/summary without ram_usage_mb: {body.memory.error}" + case _: + return f"{replica} gave no /debug/memory/summary reading: {result}" + + +def rss_capture(summaries: Mapping[str, Result[MemorySummaryResponse]]) -> RssCapture: + outcomes: Final = tuple(_outcome(replica, result) for replica, result in summaries.items()) + return RssCapture( + readings=tuple(outcome for outcome in outcomes if isinstance(outcome, RssReading)), + failures=tuple(outcome for outcome in outcomes if isinstance(outcome, str)), + ) + + +def read_rss_everywhere(proxy: ProxyClient, *, timeout: float | None = None) -> RssCapture: + return rss_capture(proxy.memory_summary_everywhere(timeout=timeout)) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 66df1c4c106..51ea9fbe7bd 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -508,13 +508,16 @@ class ProxyClient: ) ).info - def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: + def memory_summary_everywhere( + self, *, timeout: float | None = None + ) -> Mapping[str, Result[MemorySummaryResponse]]: return { url: transport.get( "/debug/memory/summary", headers=self.management_headers(transport=transport), params=NoBody(), response_type=MemorySummaryResponse, + timeout=timeout, ) for url, transport in self.replicas.items() } diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 17e3e1a1996..2568434a08f 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -38,6 +38,20 @@ size budget: the deterministic catch for a breadcrumb that copies the whole request. It runs before the phases because the leaking writer drops its own rows under the phases' traffic (a queue budget hit, a recursion limit on the nested copies), which would turn the size check into a missing-row check. + +A second, cheaper check holds the idle footprint: every worker's RSS as the harness +read it at collection time, before this pytest process sent any traffic (see +conftest.pytest_collection_finish), must sit under a fixed budget. On the +release gate that is a fresh stack right after its readiness gate, one worker per +gateway replica, so the reading is what a DB-backed boot costs on its own. A +v1.100.x worker with a database idled at 886 MB RSS where v1.101.0rc1 idled at +544 MB on the same database (1.3 GB against 560 MB at the pod level, under a +2 GiB limit): the generated Prisma client at prisma-client-py's default recursive +type depth, 91k TypedDict classes that v1.101.0 cut to 19k with +recursive_type_depth = -1. The budget starts at the rc1 reading plus headroom and +E2E_MEMORY_IDLE_RSS_BUDGET_MB overrides it; a later session on the same stack (the +changed-files workflow's repeat passes, a developer's local loop) measures a proxy +already warmed by traffic, which that headroom also has to cover. """ from __future__ import annotations @@ -55,6 +69,7 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import ( MEMORY_CONCURRENCY, + MEMORY_IDLE_RSS_BUDGET_MB, MEMORY_REQUESTS_PER_PHASE, MEMORY_RETRIES_PER_REQUEST, MEMORY_RSS_BUDGET_MB, @@ -62,10 +77,11 @@ from e2e_config import ( MEMORY_RSS_SETTLE_SAMPLES, MEMORY_STORED_REQUEST_BUDGET_KB, MEMORY_TRANSCRIPT_TURNS, + PROXY_REPLICA_URLS, unique_marker, ) -from e2e_http import unwrap from lifecycle import ResourceManager +from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment @@ -84,21 +100,6 @@ class FailedCall: call_id: str | None -WorkerKey = tuple[str, str | None, int] - - -@dataclass(frozen=True, slots=True) -class RssReading: - replica: str - hostname: str | None - worker_pid: int - ram_usage_mb: float - - @property - def worker(self) -> WorkerKey: - return (self.replica, self.hostname, self.worker_pid) - - @dataclass(frozen=True, slots=True) class WorkerGrowth: warm: RssReading @@ -143,12 +144,12 @@ def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSetting def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) - return tuple( - RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) - for replica, result in proxy.memory_summary_everywhere().items() - for body in (unwrap(result),) - if body.memory.ram_usage_mb is not None + capture: Final = read_rss_everywhere(proxy) + assert not capture.failures, ( + f"{len(capture.failures)} replica(s) gave no RSS reading mid-checkpoint, so their workers cannot be " + f"compared with themselves: {'; '.join(capture.failures)}" ) + return capture.readings def _readings_until_no_new_worker( @@ -220,6 +221,26 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.idle_memory.under_slo") + def test_workers_idle_under_rss_budget_before_traffic(self, idle_rss: RssCapture) -> None: + assert not idle_rss.failures, ( + f"{len(idle_rss.failures)} replica(s) gave no RSS reading when the session started, so their idle " + f"footprint went unmeasured: {'; '.join(idle_rss.failures)}" + ) + unmeasured: Final = frozenset(PROXY_REPLICA_URLS) - frozenset(reading.replica for reading in idle_rss.readings) + assert not unmeasured, ( + f"{len(unmeasured)} of {len(PROXY_REPLICA_URLS)} replica(s) gave neither an RSS reading nor a failure " + f"reason when the session started, so their idle footprint went unmeasured: {', '.join(sorted(unmeasured))}" + ) + heaviest: Final = idle_rss.heaviest + assert heaviest is not None, "no replica was configured to read, so nothing was measured" + assert heaviest.ram_usage_mb <= MEMORY_IDLE_RSS_BUDGET_MB, ( + f"{heaviest.where} sat at {heaviest.ram_usage_mb:.0f} MB RSS when the session started, before it sent " + f"any traffic, past the {MEMORY_IDLE_RSS_BUDGET_MB:.0f} MB idle budget; a DB-backed v1.100.x worker idled " + f"at 886 MB where v1.101.0rc1 idled at 544 MB, and at that size the release stack's 2 GiB pod limit " + f"leaves the worker little room for traffic" + ) + @pytest.mark.covers("reliability.perf.memory.under_slo") def test_failing_requests_do_not_grow_rss_or_stored_request( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str From d31e8aac6d41819fa3f4de739b7995ca7c337264 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:48:07 +0000 Subject: [PATCH 03/17] feat(cost-map): add Claude Opus 5.5 for Vertex AI and Azure AI (#42599) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 107 ++++++++++++++++++ model_prices_and_context_window.json | 107 ++++++++++++++++++ .../test_litellm/test_claude_opus_5_config.py | 36 ++++-- 3 files changed, 242 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 34cbea0e314..3f1d26e9adc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3527,6 +3527,41 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "azure_ai/claude-opus-5-5": { + "supports_mid_conversation_system": true, + "input_cost_per_token": 4e-06, + "output_cost_per_token": 2e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "supports_mid_conversation_system": true, @@ -47016,6 +47051,78 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "vertex_ai/claude-opus-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, + "vertex_ai/claude-opus-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", "regional_endpoint_uplift_multiplier": 1.1, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 34cbea0e314..3f1d26e9adc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3527,6 +3527,41 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "azure_ai/claude-opus-5-5": { + "supports_mid_conversation_system": true, + "input_cost_per_token": 4e-06, + "output_cost_per_token": 2e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "supports_mid_conversation_system": true, @@ -47016,6 +47051,78 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "vertex_ai/claude-opus-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, + "vertex_ai/claude-opus-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", "regional_endpoint_uplift_multiplier": 1.1, diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index 2e0a359b79b..b5efe016a60 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -41,7 +41,10 @@ ALL_OPUS_5_VARIANTS = ( "jp.anthropic.claude-opus-5", "vertex_ai/claude-opus-5", "vertex_ai/claude-opus-5@default", + "vertex_ai/claude-opus-5-5", + "vertex_ai/claude-opus-5-5@default", "azure_ai/claude-opus-5", + "azure_ai/claude-opus-5-5", ) BEDROCK_OPUS_5_VARIANTS = ( @@ -69,20 +72,37 @@ def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS -def test_opus_5_5_present_in_bundled_backup(): +OPUS_5_5_VARIANTS = ( + "claude-opus-5-5", + "vertex_ai/claude-opus-5-5", + "vertex_ai/claude-opus-5-5@default", + "azure_ai/claude-opus-5-5", +) + + +@pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS) +def test_opus_5_5_present_in_bundled_backup(model_name): backup = GetModelCostMap.load_local_model_cost_map() root = _load_root_cost_map() - assert "claude-opus-5-5" in backup - assert "claude-opus-5-5" in root - assert backup["claude-opus-5-5"] == root["claude-opus-5-5"] + assert model_name in backup + assert model_name in root + assert backup[model_name] == root[model_name] -@pytest.mark.parametrize("model", ["claude-opus-5-5", "anthropic/claude-opus-5-5"]) -def test_opus_5_5_thinking_profile(local_model_cost_map, model): +@pytest.mark.parametrize( + ("model", "provider"), + [ + ("claude-opus-5-5", "anthropic"), + ("anthropic/claude-opus-5-5", "anthropic"), + ("vertex_ai/claude-opus-5-5", "vertex_ai"), + ("azure_ai/claude-opus-5-5", "azure_ai"), + ], +) +def test_opus_5_5_thinking_profile(local_model_cost_map, model, provider): """Opus 5.5 has thinking always on with the adaptive thinking surface, and no forced tool use, same as Fable 5.1.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True - assert AnthropicModelInfo._is_always_on_thinking_model(model, "anthropic") is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, provider) is True + assert AnthropicModelInfo._is_always_on_thinking_model(model, provider) is True assert AnthropicModelInfo.forced_tool_use_unsupported(model.removeprefix("anthropic/")) is True From d7c27cdc087937abfade549b2bb5fa9e968a8081 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:52:01 -0700 Subject: [PATCH 04/17] feat(proxy): configurable key_alias_pattern for key generate, update, and regenerate (#42553) * feat(proxy): configurable key_alias_pattern for key generate, update, and regenerate Adds litellm_settings.key_alias_pattern, a regex every key_alias sent to /key/generate, /key/service-account/generate, /key/update, and /key/{key}/regenerate has to fully match. A non-matching alias gets a 400 that names the setting and the pattern. When set, it replaces the built-in rule enable_key_alias_format_validation turns on, and the baseline unsafe-name check still runs first. An invalid regex fails config load. * fix(proxy): cap key_alias length under key_alias_pattern and type the test fixtures * style(proxy): declare key_alias_pattern with a PEP 604 union --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 1 + .../key_management_endpoints.py | 56 +++++-- litellm/proxy/proxy_server.py | 3 + .../test_key_management_endpoints.py | 158 ++++++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 17 ++ 5 files changed, 219 insertions(+), 16 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index a044676a843..471b273f00d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -381,6 +381,7 @@ enable_model_config_credential_overrides: bool = False enable_key_alias_format_validation: bool = ( False # opt-in validation of key_alias format on /key/generate and /key/update ) +key_alias_pattern: str | None = None enable_gemini_default_thinking_level_low: bool = ( False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 801c45501ce..1845f11c0fa 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -7590,25 +7590,47 @@ async def test_key_logging( _KEY_ALIAS_PATTERN: Final = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_\-/\.@]{0,253}[a-zA-Z0-9]$") +_KEY_ALIAS_PATTERN_MESSAGE: Final = ( + "Invalid key_alias format. Must be 2-255 characters, start/end with alphanumeric, and only contain a-zA-Z0-9_-/.@." +) +_KEY_ALIAS_MAX_LENGTH: Final = 255 + + +def parse_key_alias_pattern(value: object) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError( + f"Invalid regex set for litellm_settings.key_alias_pattern - value={value!r}: must be a string" + ) + try: + re.compile(value) + except re.error as e: + raise ValueError(f"Invalid regex set for litellm_settings.key_alias_pattern - value={value}: {e}") from e + return value + + +def _key_alias_rule() -> tuple[re.Pattern[str], str] | None: + if litellm.key_alias_pattern is not None: + return ( + re.compile(litellm.key_alias_pattern), + f"Invalid key_alias format. Must be at most {_KEY_ALIAS_MAX_LENGTH} characters and match the configured" + f" key_alias_pattern: {litellm.key_alias_pattern}", + ) + if litellm.enable_key_alias_format_validation: + return (_KEY_ALIAS_PATTERN, _KEY_ALIAS_PATTERN_MESSAGE) + return None def _validate_key_alias_format(key_alias: str | None) -> None: """ Validate the format of the key_alias. - A baseline validation always runs, regardless of - ``litellm.enable_key_alias_format_validation``. - - The remaining charset/length rules are gated behind - ``litellm.enable_key_alias_format_validation`` (default **False**). When disabled, - only the baseline validation above is performed, so existing workflows are not - broken. - - Rules (when enabled): - - None is OK (no alias). - - Otherwise must be 2–255 chars - - start/end with alphanumeric - - only allow a-zA-Z0-9_-/.@ + Path traversal and control characters are always rejected. The alias then has to + stay within ``_KEY_ALIAS_MAX_LENGTH`` and fully match ``litellm.key_alias_pattern`` + when one is configured, else the built-in pattern when + ``litellm.enable_key_alias_format_validation`` is on, else nothing more is checked + so existing workflows are not broken. """ if key_alias is None: return @@ -7623,12 +7645,14 @@ def _validate_key_alias_format(key_alias: str | None) -> None: code=400, ) - if not litellm.enable_key_alias_format_validation: + rule: Final = _key_alias_rule() + if rule is None: return - if not _KEY_ALIAS_PATTERN.match(key_alias): + pattern, message = rule + if len(key_alias) > _KEY_ALIAS_MAX_LENGTH or pattern.fullmatch(key_alias) is None: raise ProxyException( - message="Invalid key_alias format. Must be 2-255 characters, start/end with alphanumeric, and only contain a-zA-Z0-9_-/.@.", + message=message, type=ProxyErrorTypes.bad_request_error, param="key_alias", code=400, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 10110dd4f76..acb36fc4a59 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -596,6 +596,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, generate_key_helper_fn, + parse_key_alias_pattern, ) from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, @@ -6266,6 +6267,8 @@ class ProxyConfig: litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams(**value) else: raise Exception(f"Invalid value set for upperbound_key_generate_params - value={value}") + elif key == "key_alias_pattern": + litellm.key_alias_pattern = parse_key_alias_pattern(value) elif key == "json_logs" and value is True: litellm.json_logs = True litellm._turn_on_json() diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index c539a10e1e2..cd929dfeeb3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -3089,6 +3089,50 @@ async def test_update_key_by_alias_only(monkeypatch): assert result["key"] == hashed_token +@pytest.mark.asyncio +async def test_update_key_changed_alias_must_match_key_alias_pattern(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + update_key_fn, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + hashed_token = "0d62f396c1317066f55a96086517047c737087c61eb2bf016b72e6298927b15b" + key_in_db = LiteLLM_VerificationToken(token=hashed_token, key_alias="Legacy Alias", user_id="test-user") + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_in_db) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_in_db]) + mock_prisma_client.db.litellm_verificationtoken.find_first = AsyncMock(return_value=None) + mock_prisma_client.update_data = AsyncMock(return_value={"data": {"max_budget": 50.0}}) + _setup_update_key_mocks(monkeypatch, mock_prisma_client) + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin-user" + ) + + with pytest.raises(ProxyException) as exc_info: + await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key=hashed_token, key_alias="Prod Key"), + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + assert str(exc_info.value.code) == "400" + assert "key_alias_pattern" in str(exc_info.value.message) + mock_prisma_client.update_data.assert_not_awaited() + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + return_value=None, + ): + await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key=hashed_token, key_alias="Legacy Alias", max_budget=50.0), + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + mock_prisma_client.update_data.assert_awaited_once() + + @pytest.mark.asyncio async def test_update_key_by_alias_not_found_returns_404(monkeypatch): """ @@ -10562,6 +10606,10 @@ class TestValidateKeyAliasFormat: def reset_key_alias_flag(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "enable_key_alias_format_validation", False) + @pytest.fixture(autouse=True) + def reset_key_alias_pattern(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "key_alias_pattern", None) + def test_validation_skipped_when_flag_disabled(self): """When enable_key_alias_format_validation is False (default), no charset/length validation occurs.""" from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -10644,6 +10692,67 @@ class TestValidateKeyAliasFormat: assert str(exc.value.code) == "400" assert "Invalid key_alias format" in str(exc.value.message) + def test_configured_pattern_applies_with_flag_off(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("Prod Key") + assert str(exc.value.code) == "400" + assert exc.value.param == "key_alias" + assert "key_alias_pattern" in str(exc.value.message) + assert r"^[a-z0-9]+(-[a-z0-9]+)*$" in str(exc.value.message) + assert _validate_key_alias_format("prod-key-001") is None + assert _validate_key_alias_format(None) is None + + def test_configured_pattern_must_match_the_whole_alias(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"team-[a-z]+") + _validate_key_alias_format("team-search") + for partial_match in ("team-search-2", "xteam-search"): + with pytest.raises(ProxyException): + _validate_key_alias_format(partial_match) + + def test_configured_pattern_replaces_the_builtin_rule(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z ]+$") + _validate_key_alias_format("alias with spaces") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("Uppercase") + assert "key_alias_pattern" in str(exc.value.message) + + def test_configured_pattern_keeps_the_baseline_safety_check(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r".*") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("../../../other-app/creds") + assert str(exc.value.code) == "400" + assert "key_alias_pattern" not in str(exc.value.message) + + def test_configured_pattern_bounds_the_alias_length(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z]+$") + _validate_key_alias_format("a" * 255) + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("a" * 256) + assert str(exc.value.code) == "400" + assert "at most 255 characters" in str(exc.value.message) + @pytest.mark.asyncio async def test_check_org_key_limits_on_update_within_bounds(): @@ -12734,6 +12843,55 @@ async def test_execute_virtual_key_regeneration_rejects_over_limit_duration(monk assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 +@pytest.mark.asyncio +async def test_execute_virtual_key_regeneration_changed_alias_must_match_key_alias_pattern( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import RegenerateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _execute_virtual_key_regeneration, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + mock_prisma_client = _make_regenerate_mock_prisma() + + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._persist_deleted_verification_tokens", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=_make_regenerate_existing_key(), + hashed_api_key="abc123", + key="abc123", + data=RegenerateKeyRequest(key_alias="Regenerated Key"), + user_api_key_dict=_make_regenerate_user_api_key_dict(), + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert str(exc_info.value.code) == "400" + assert exc_info.value.param == "key_alias" + assert r"^[a-z0-9]+(-[a-z0-9]+)*$" in str(exc_info.value.message) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 + + @pytest.mark.asyncio async def test_execute_virtual_key_regeneration_allows_within_limit_duration(monkeypatch): """Regenerate must accept durations within upperbound_key_generate_params.duration.""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 32b89885ac2..89fd9c5c9d4 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3555,6 +3555,23 @@ async def test_load_config_rejects_malformed_role_permissions(tmp_path): await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) +@pytest.mark.asyncio +async def test_load_config_compiles_key_alias_pattern_at_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr(litellm, "key_alias_pattern", None) + config_file: Final = tmp_path / "config.yaml" + + config_file.write_text(yaml.dump({"model_list": [], "litellm_settings": {"key_alias_pattern": "^team-("}})) + with pytest.raises(Exception, match=r"litellm_settings\.key_alias_pattern"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert litellm.key_alias_pattern is None + + config_file.write_text(yaml.dump({"model_list": [], "litellm_settings": {"key_alias_pattern": "^team-[a-z]+$"}})) + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert litellm.key_alias_pattern == "^team-[a-z]+$" + + def test_os_environ_resolution_leaves_the_config_layer_holding_the_reference(monkeypatch): from litellm.proxy.proxy_server import ProxyConfig From 65e42526d6a2dd7875e3f8618b7dd2d2d387028c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 22 Sep 2026 14:58:42 -0700 Subject: [PATCH 05/17] test(proxy): make two proxy-infra tests independent of sibling-test state (#42581) * test(proxy): make two proxy-infra tests independent of sibling-test state Both tests read process-global state that another module in the same xdist worker can change, so they passed or failed on shard scheduling rather than on the behaviour they assert. test_gateway_plus_backend_covers_full_app checked allowlist coverage against the live route table. gateway/main.py trims routes once, inside the lifespan, so it only ever sees what is registered at startup; a lazy feature appends its router on demand afterwards and is never filtered. Those routes cannot be dropped on the floor, but they do enter the assertion the moment a sibling test warms the feature, and 110 of them sit in neither allowlist. Subtract exactly the lazy features this process has loaded, which leaves the assertion at full strength for every eagerly registered route. test_real_proxy_child_auth_privacy_and_body_policy pins prisma_client to a bare object(). litellm.max_budget is a module global that nothing restores between tests; once a sibling leaves it above zero, user_api_key_auth takes the global-spend branch, dereferences prisma_client.db, and the AttributeError surfaces as HTTP 401. Pin max_budget next to the other globals the test already controls. * test(proxy): measure allowlist coverage in a pristine interpreter The previous revision subtracted paths matching a loaded lazy feature's prefixes. Those prefixes are broad enough to swallow eagerly registered routes: 31 of them, including /openai/deployments/*, /access_group/*, /cursor/* and /mcp, which would have made a real allowlist regression invisible. Run the coverage check in a fresh interpreter instead. No lazy feature is loaded there, so the route table is exactly the one gateway/main.py's lifespan trim sees, and nothing has to be subtracted for the result to be deterministic. The probe also reports the lazy modules it loaded and its route count, so an empty uncovered set cannot pass vacuously. Drop _component_paths and the four allowlist constants it used; the probe reproduces the predicate in the child process. --- .../proxy/test_component_allowlists.py | 94 ++++++++++++------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/tests/test_litellm/proxy/test_component_allowlists.py b/tests/test_litellm/proxy/test_component_allowlists.py index 3073908fa54..3641a2d9be9 100644 --- a/tests/test_litellm/proxy/test_component_allowlists.py +++ b/tests/test_litellm/proxy/test_component_allowlists.py @@ -23,8 +23,10 @@ import (which raises on a non-postgres ``DATABASE_URL`` scheme and can mint an RDS IAM token when ``IAM_TOKEN_DB_AUTH`` is set). """ +import json import os import sys +from typing import Final # Importing ``litellm.proxy.proxy_server`` runs its module-level setup, which # reads ``DATABASE_URL`` (Prisma) and ``LITELLM_MASTER_KEY``. Tier-zero CI @@ -49,17 +51,10 @@ _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) -from backend.routes.allowlist import ( - BACKEND_EXACT_PATHS, - BACKEND_MOUNT_PATHS, - BACKEND_PATH_PREFIXES, -) -from gateway.routes.allowlist import ( - GATEWAY_EXACT_PATHS, - GATEWAY_MOUNT_PATHS, - GATEWAY_PATH_PREFIXES, -) +from backend.routes.allowlist import BACKEND_MOUNT_PATHS +from gateway.routes.allowlist import GATEWAY_MOUNT_PATHS from litellm.proxy.proxy_server import app +from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter for _key, _previous in _PRE_EXISTING_ENV.items(): if _previous is None: @@ -87,40 +82,69 @@ for _key, _previous in _PRE_DB_ENV.items(): os.environ[_key] = _previous -def _component_paths(routes, exact_paths, path_prefixes) -> set[str]: - """Reproduce ``gateway.main._is_gateway_route`` / ``backend.main._is_backend_route``.""" - out: set[str] = set() - for route in routes: - if isinstance(route, Mount): - continue - path = getattr(route, "path", None) - if path is None: - continue - if path in exact_paths or any(path.startswith(p) for p in path_prefixes): - out.add(path) - return out +_COVERAGE_PROBE: Final = """ +import json, os, sys +sys.path.insert(0, os.environ["LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT"]) +from fastapi.routing import Mount +from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES +from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES +from litellm.proxy._lazy_features import loaded_lazy_modules +from litellm.proxy.proxy_server import app + +all_paths = { + r.path for r in app.router.routes + if not isinstance(r, Mount) and getattr(r, "path", None) is not None +} + + +def covered(exact, prefixes): + return {p for p in all_paths if p in exact or any(p.startswith(x) for x in prefixes)} + + +json.dump({ + "lazy_loaded": sorted(loaded_lazy_modules(app)), + "route_count": len(all_paths), + "uncovered": sorted(all_paths - ( + covered(GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES) + | covered(BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES) + )), +}, sys.stdout) +""" def test_gateway_plus_backend_covers_full_app(): - """Every route on the proxy app must be served by gateway or backend.""" - all_paths = { - getattr(r, "path") - for r in app.router.routes - if not isinstance(r, Mount) and getattr(r, "path", None) is not None - } - gateway_paths = _component_paths( - app.router.routes, GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES + """Every route on the proxy app must be served by gateway or backend. + + ``gateway.main`` and ``backend.main`` trim the route table once, inside the + lifespan, so the set this has to cover is the one registered at startup. A + lazy feature appends its router on demand, after that trim, and whether a + sibling test in the same xdist worker has triggered one is not something + this test can control. Measuring in a fresh interpreter is what makes the + route table deterministic; nothing is subtracted, so every route the trim + will actually see stays in the assertion. + """ + env: Final = {**os.environ, "LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT": _REPO_ROOT} + for key, value in _THROWAWAY_ENV.items(): + env.setdefault(key, value) + + probe: Final = run_child_interpreter(_COVERAGE_PROBE, env=env, timeout=90) + assert probe.returncode == 0, f"route probe failed:\n{probe.stderr}" + report: Final = json.loads(probe.stdout) + + assert not report["lazy_loaded"], ( + "route probe was not pristine; it loaded lazy features " + f"{report['lazy_loaded']}, so its route table is not the startup one" ) - backend_paths = _component_paths( - app.router.routes, BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES + assert report["route_count"] > 100, ( + f"route probe only saw {report['route_count']} routes, so an empty " + "uncovered set would not mean anything" ) - uncovered = all_paths - (gateway_paths | backend_paths) - + uncovered: Final = report["uncovered"] assert not uncovered, ( f"{len(uncovered)} route(s) are not exposed on either component. " f"Update gateway/routes/allowlist.py or backend/routes/allowlist.py to cover:\n " - + "\n ".join(sorted(uncovered)) + + "\n ".join(uncovered) ) From 0d6ee3dc5a85e6c49caa2eb41660fc5c33cdfc7d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:59:26 -0700 Subject: [PATCH 06/17] test: count a zombie grandchild as gone in the migrate deploy timeout test (#42570) * test: count a zombie grandchild as gone in the migrate deploy timeout test A SIGKILLed grandchild whose parent died in the same killpg reparents to PID 1 or the nearest subreaper and stays a zombie until reaped, and signal 0 still succeeds on a zombie, so the timeout test read it as alive wherever PID 1 is slow to reap or never does. The sibling test in tests/test_litellm/proxy/db already handled that; both now share one process_is_gone helper that reads the /proc state and reaps its own children, with unit tests for the live, reaped, unreaped, and foreign zombie shapes. * test: move the pre-commit interrupt test onto the shared zombie-aware liveness helper --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/_process_helpers.py | 57 ++++++++++++++++ .../test_prisma_toolchain.py | 18 ++--- tests/test_litellm/proxy/db/conftest.py | 32 ++------- tests/test_litellm/test_pre_commit_lint.py | 12 +--- tests/test_litellm/test_process_helpers.py | 65 +++++++++++++++++++ 5 files changed, 133 insertions(+), 51 deletions(-) create mode 100644 tests/_process_helpers.py create mode 100644 tests/test_litellm/test_process_helpers.py diff --git a/tests/_process_helpers.py b/tests/_process_helpers.py new file mode 100644 index 00000000000..9f80c563fa9 --- /dev/null +++ b/tests/_process_helpers.py @@ -0,0 +1,57 @@ +"""Whether a killed process is really gone, for tests that kill whole process trees. + +A SIGKILLed grandchild whose parent died in the same ``killpg`` reparents to the +nearest subreaper or PID 1, and until that ancestor reaps it the pid is a zombie +that ``os.kill(pid, 0)`` still accepts. Reading its ``/proc`` state, and reaping +it when it landed on this process, keeps a runner that is slow to reap, or never +does, from turning a dead process into a failed assertion. The reap comes after +the liveness read so a child seen dying between the two is still collected on +the next poll instead of staying this process's own zombie. +""" + +import os +import time +from pathlib import Path +from typing import Final + +POLL_INTERVAL_S: Final = 0.05 + + +def _exists(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +def _is_zombie(pid: int) -> bool: + try: + stat: Final = Path(f"/proc/{pid}/stat").read_text() + except OSError: + return False + return stat.rpartition(")")[2].split()[0] == "Z" + + +def _reap_if_ours(pid: int) -> None: + if os.name == "nt": + return + try: + os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + pass + + +def _gone_now(pid: int) -> bool: + dead: Final = not _exists(pid) or _is_zombie(pid) + _reap_if_ours(pid) + return dead + + +def process_is_gone(pid: int, within_seconds: float) -> bool: + deadline: Final = time.monotonic() + within_seconds + while time.monotonic() < deadline: + if _gone_now(pid): + return True + time.sleep(POLL_INTERVAL_S) + return False diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index 4e2274cc582..556c680a84a 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -25,7 +25,6 @@ from collections.abc import Callable from pathlib import Path import pytest - from litellm_proxy_extras.prisma_toolchain import ( DEFAULT_PRISMA_COMMAND_TIMEOUT, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, @@ -36,14 +35,16 @@ from litellm_proxy_extras.prisma_toolchain import ( heal_incomplete_nodeenv_cache, node_binary_path, prisma_bootstrap_timeout, - prisma_command_timeout, prisma_cli_available, + prisma_command_timeout, prisma_migrate_deploy_timeout, resolve_prisma_argv, run_prisma, ) from litellm_proxy_extras.utils import ProxyExtrasDBManager +from tests._process_helpers import process_is_gone + REPO_ROOT = Path(__file__).resolve().parents[2] PROXY_EXTRAS = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" @@ -280,17 +281,6 @@ def test_migrate_deploy_stops_at_its_own_timeout( assert elapsed < 30 -def _process_is_gone(pid: int, within_seconds: float) -> bool: - deadline = time.monotonic() + within_seconds - while time.monotonic() < deadline: - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - time.sleep(0.05) - return False - - def test_a_timed_out_migrate_deploy_takes_its_process_tree_with_it( toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -309,7 +299,7 @@ def test_a_timed_out_migrate_deploy_takes_its_process_tree_with_it( grandchild_pid = int(pidfile.read_text()) try: assert len(_deploy_calls(log_path)) == 2 - assert _process_is_gone(grandchild_pid, within_seconds=5) + assert process_is_gone(grandchild_pid, within_seconds=5) finally: try: os.kill(grandchild_pid, signal.SIGKILL) diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py index bcb7794a20d..4d775142dd5 100644 --- a/tests/test_litellm/proxy/db/conftest.py +++ b/tests/test_litellm/proxy/db/conftest.py @@ -2,14 +2,15 @@ import json import os import signal import sys -import time from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import Final, Optional +from typing import Optional import pytest +from tests._process_helpers import process_is_gone + DB_ENV_KEYS = ( "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", @@ -35,14 +36,6 @@ DB_ENV_KEYS = ( _db_env_snapshot_key = pytest.StashKey[dict[str, Optional[str]]]() -def _is_zombie(pid: int) -> bool: - try: - stat: Final = Path(f"/proc/{pid}/stat").read_text() - except OSError: - return False - return stat.rpartition(")")[2].split()[0] == "Z" - - def _db_env_snapshot() -> dict[str, Optional[str]]: return {key: os.environ.get(key) for key in DB_ENV_KEYS} @@ -130,24 +123,7 @@ class FakePrismaCli: return [json.loads(line) for line in self.calls_file.read_text().splitlines()] def grandchild_is_gone(self, within_seconds: float) -> bool: - pid: Final = int(self.grandchild_pidfile.read_text()) - deadline: Final = time.monotonic() + within_seconds - while time.monotonic() < deadline: - if os.name != "nt": - try: - reaped_pid, _ = os.waitpid(pid, os.WNOHANG) - if reaped_pid == pid: - return True - except ChildProcessError: - pass - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - if _is_zombie(pid): - return True - time.sleep(0.05) - return False + return process_is_gone(int(self.grandchild_pidfile.read_text()), within_seconds=within_seconds) @pytest.fixture diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index e12da0833dc..2d7fa897536 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -9,6 +9,8 @@ from pathlib import Path import pytest +from tests._process_helpers import process_is_gone + ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "pre_commit_lint.sh" WHOLE_TREE_RUFF = "run --no-sync ruff check --config ruff-tests.toml tests" @@ -343,14 +345,6 @@ def _wait_until(predicate: Callable[[], bool], timeout_seconds: float) -> bool: return predicate() -def _pid_gone(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - return False - - def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) hang_dir = tmp_path / "hang" @@ -372,7 +366,7 @@ def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> Non os.killpg(proc.pid, signal.SIGINT) assert proc.wait(timeout=10) != 0 make_pid = int((hang_dir / "make.pid").read_text()) - assert _wait_until(lambda: _pid_gone(make_pid), 5) + assert process_is_gone(make_pid, within_seconds=5) assert _wait_until(lambda: not any(tmp_dir.iterdir()), 5), list(tmp_dir.iterdir()) finally: with suppress(ProcessLookupError, PermissionError): diff --git a/tests/test_litellm/test_process_helpers.py b/tests/test_litellm/test_process_helpers.py new file mode 100644 index 00000000000..00d4d07d71f --- /dev/null +++ b/tests/test_litellm/test_process_helpers.py @@ -0,0 +1,65 @@ +"""``process_is_gone`` has to say gone for every shape a killed process can take, and never for a live one.""" + +import os +import signal +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest + +from tests._process_helpers import process_is_gone + +SLEEP_FOREVER: Final = (sys.executable, "-I", "-c", "import time; time.sleep(600)") + +LEAVE_A_ZOMBIE_BEHIND: Final = """ +import os, signal, subprocess, sys, time +grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]) +os.kill(grandchild.pid, signal.SIGKILL) +while open(f"/proc/{grandchild.pid}/stat").read().rpartition(")")[2].split()[0] != "Z": + time.sleep(0.01) +print(grandchild.pid, flush=True) +time.sleep(600) +""" + + +def test_a_live_process_is_not_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + try: + assert not process_is_gone(child.pid, within_seconds=0.3) + finally: + child.kill() + child.wait() + + +def test_a_reaped_child_is_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + child.kill() + child.wait() + assert process_is_gone(child.pid, within_seconds=1) + + +@pytest.mark.skipif(os.name == "nt", reason="zombies are a POSIX thing") +def test_an_unreaped_child_is_reaped_and_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + os.kill(child.pid, signal.SIGKILL) + assert process_is_gone(child.pid, within_seconds=1) + with pytest.raises(ChildProcessError): + os.waitpid(child.pid, os.WNOHANG) + + +@pytest.mark.skipif(not Path("/proc").is_dir(), reason="needs procfs to see a zombie that is not our child") +def test_a_zombie_left_by_another_process_is_gone() -> None: + zombie_factory: Final = subprocess.Popen( + [sys.executable, "-I", "-c", LEAVE_A_ZOMBIE_BEHIND], stdout=subprocess.PIPE, text=True + ) + try: + assert zombie_factory.stdout is not None + zombie_pid: Final = int(zombie_factory.stdout.readline()) + with pytest.raises(ChildProcessError): + os.waitpid(zombie_pid, os.WNOHANG) + assert process_is_gone(zombie_pid, within_seconds=1) + finally: + zombie_factory.kill() + zombie_factory.wait() From b9fcfb26d01aaba21316ff9cb2ab1f4b2abf0fe4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:13:37 -0700 Subject: [PATCH 07/17] test(e2e): run the memory cell alone on the shared stack (#42518) * test(e2e): run the memory cell alone on the shared stack * test(e2e): hold the stack lock for every collected test, marker or not * test(e2e): prove the stack lock's reader sharing and writer preference across processes --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- tests/e2e/conftest.py | 20 ++- tests/e2e/pytest.ini | 1 + .../e2e/router/test_reliability_memory_e2e.py | 2 +- tests/e2e/stack_lock.py | 45 +++++++ tests/e2e/test_stack_lock.py | 117 ++++++++++++++++++ 6 files changed, 179 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/stack_lock.py create mode 100644 tests/e2e/test_stack_lock.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 6212a172fe3..d948c6fd1d9 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -97,7 +97,7 @@ Each suite provides its own `client` fixture (see `llm_translation/passthrough_c Request and response bodies are typed pydantic models in `models.py`; only the fields a test reads are modelled, and nothing passes raw dicts. Outcomes come back as a `Result[R]` tagged union (`Success`, `NetworkError`, `UnauthorizedError`, `RateLimitedError`, `ValidationError`, `UnknownApiError`). Handle them with `match`, or call `unwrap(...)` when a non-success should fail the test. The harness hard-fails and never skips: a test marked `e2e` fails when no proxy answers its liveness probe, and once a request reaches the proxy any wrong behavior is likewise a hard failure, so a missing proxy turns the run red instead of being mistaken for a pass -Mark live tests with `@pytest.mark.e2e` (on the class or the module). Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +Mark live tests with `@pytest.mark.e2e` (on the class or the module). Add `@pytest.mark.quiet_stack` to a test that measures the proxy itself (RSS, latency): the shared stack lock in `stack_lock.py` then runs it while no other test on the host is hitting the stack, marked or not, so the reading depends only on the test's own traffic. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache ## Record and replay fixtures diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index f1f94f2d2b3..f4de4ac01ba 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -51,6 +51,7 @@ from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client +from stack_lock import stack_lock _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() @@ -148,6 +149,11 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) + config.addinivalue_line( + "markers", + "quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; " + "every other test waits for it to finish", + ) config.addinivalue_line( "markers", "mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless " @@ -172,9 +178,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: """Abort before collection when E2E_FIXTURE_MODE can never work: an unknown mode value, or replay against a missing, unreadable, or stale bundle (the stale message names the bundle's age). Live and record modes pass through.""" - reason = fixture_mode_collection_error( - FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc) - ) + reason = fixture_mode_collection_error(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)) if reason is not None: raise pytest.UsageError(reason) @@ -267,6 +271,12 @@ def _proxy_fail_reason() -> str | None: return None +@pytest.hookimpl(wrapper=True) +def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None) -> Generator[None, object, object]: + with stack_lock(exclusive=item.get_closest_marker("quiet_stack") is not None): + return (yield) + + @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. @@ -326,9 +336,7 @@ def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: LIVE_PROVIDER_REQUIRED.set(False) if not item.stash.get(_CALL_PASSED, False): return result - reason = replay_leftover_error( - mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid - ) + reason = replay_leftover_error(mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid) if reason is not None: pytest.fail(reason) return result diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index a77459d3683..4866511af3f 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -12,6 +12,7 @@ markers = prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 2568434a08f..77d3a68cae5 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -86,7 +86,7 @@ from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] DEPLOYMENTS_PER_GROUP: Final = 2 RSS_SAMPLE_CAP: Final = 4 * MEMORY_RSS_SETTLE_SAMPLES diff --git a/tests/e2e/stack_lock.py b/tests/e2e/stack_lock.py new file mode 100644 index 00000000000..06df7a20b6a --- /dev/null +++ b/tests/e2e/stack_lock.py @@ -0,0 +1,45 @@ +"""Cross-process reader/writer lock over the proxy stack every xdist worker shares. +Every collected test holds it shared, marker or not, since the Claude Code cells and +other unmarked suites drive the same stack; a `quiet_stack` test holds it exclusive, +and the `gate` file makes a waiting exclusive holder win over readers that arrive +after it.""" + +from __future__ import annotations + +import fcntl +import hashlib +import tempfile +from collections.abc import Generator +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Final + +from e2e_config import PROXY_BASE_URL + +STACK_DIGEST: Final = hashlib.sha256(PROXY_BASE_URL.encode()).hexdigest()[:12] +LOCK_DIR: Final = Path(tempfile.gettempdir()) / f"litellm-e2e-stack-{STACK_DIGEST}" +GATE_FILE: Final = LOCK_DIR / "gate" +STACK_FILE: Final = LOCK_DIR / "stack" + + +@contextmanager +def _flock(path: Path, operation: int) -> Generator[None]: + with path.open("a") as handle: + fcntl.flock(handle, operation) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +@contextmanager +def stack_lock(exclusive: bool) -> Generator[None]: + LOCK_DIR.mkdir(parents=True, exist_ok=True) + if exclusive: + with _flock(GATE_FILE, fcntl.LOCK_EX), _flock(STACK_FILE, fcntl.LOCK_EX): + yield + return + with ExitStack() as held: + with _flock(GATE_FILE, fcntl.LOCK_SH): + held.enter_context(_flock(STACK_FILE, fcntl.LOCK_SH)) + yield diff --git a/tests/e2e/test_stack_lock.py b/tests/e2e/test_stack_lock.py new file mode 100644 index 00000000000..af071d2cc18 --- /dev/null +++ b/tests/e2e/test_stack_lock.py @@ -0,0 +1,117 @@ +"""Cross-process behavior of the stack lock: readers share it, an exclusive holder waits for +every reader and keeps them out, and a reader arriving behind a waiting exclusive holder +queues behind it instead of starving it.""" + +from __future__ import annotations + +import fcntl +import os +import subprocess +import sys +import time +from contextlib import ExitStack +from pathlib import Path +from typing import Final + +import pytest + +from stack_lock import STACK_DIGEST + +HARNESS_DIR: Final = Path(__file__).resolve().parent +DEADLINE_SECONDS: Final = 30.0 +SETTLE_SECONDS: Final = 0.5 +HOLDER_SCRIPT: Final = """ +import sys, time +from pathlib import Path +from stack_lock import stack_lock +name, mode, release_path, log_path = sys.argv[1:] + + +def record(event): + with Path(log_path).open("a") as log: + log.write(f"{name} {event}\\n") + + +record("waiting") +with stack_lock(exclusive=mode == "exclusive"): + record("enter") + while not Path(release_path).exists(): + time.sleep(0.02) + record("exit") +""" + + +def _events(log_path: Path) -> tuple[str, ...]: + return tuple(log_path.read_text().splitlines()) if log_path.exists() else () + + +def _wait_for_event(log_path: Path, event: str) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + while event not in _events(log_path): + if time.monotonic() > deadline: + pytest.fail(f"{event!r} never appeared; events so far: {_events(log_path)}") + time.sleep(0.02) + + +def _wait_until_gate_is_held_exclusively(gate_path: Path) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + with gate_path.open("a") as handle: + while True: + try: + fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError: + return + fcntl.flock(handle, fcntl.LOCK_UN) + if time.monotonic() > deadline: + pytest.fail("no exclusive holder ever took the gate") + time.sleep(0.02) + + +def _start_holder(held: ExitStack, tmp_path: Path, name: str, mode: str) -> subprocess.Popen[bytes]: + holder: Final = held.enter_context( + subprocess.Popen( + ( + sys.executable, + "-P", + "-c", + HOLDER_SCRIPT, + name, + mode, + str(tmp_path / f"release-{name}"), + str(tmp_path / "events"), + ), + cwd=HARNESS_DIR, + env={**os.environ, "TMPDIR": str(tmp_path), "PYTHONPATH": str(HARNESS_DIR)}, + ) + ) + held.callback(holder.kill) + return holder + + +def test_readers_share_exclusive_waits_and_a_waiting_exclusive_beats_later_readers(tmp_path: Path) -> None: + lock_dir: Final = tmp_path / f"litellm-e2e-stack-{STACK_DIGEST}" + lock_dir.mkdir() + log_path: Final = tmp_path / "events" + with ExitStack() as held: + first_reader: Final = _start_holder(held, tmp_path, "A", "shared") + _wait_for_event(log_path, "A enter") + second_reader: Final = _start_holder(held, tmp_path, "R", "shared") + _wait_for_event(log_path, "R enter") + (tmp_path / "release-R").touch() + _wait_for_event(log_path, "R exit") + writer: Final = _start_holder(held, tmp_path, "W", "exclusive") + _wait_until_gate_is_held_exclusively(lock_dir / "gate") + late_reader: Final = _start_holder(held, tmp_path, "B", "shared") + _wait_for_event(log_path, "B waiting") + time.sleep(SETTLE_SECONDS) + (tmp_path / "release-A").touch() + _wait_for_event(log_path, "W enter") + (tmp_path / "release-W").touch() + _wait_for_event(log_path, "B enter") + (tmp_path / "release-B").touch() + for holder in (first_reader, second_reader, writer, late_reader): + assert holder.wait(timeout=DEADLINE_SECONDS) == 0 + events: Final = _events(log_path) + assert events.index("R enter") < events.index("A exit") + assert events.index("W enter") > events.index("A exit") + assert events.index("B enter") > events.index("W exit") From a173657dfb2df85511d07febd1f15adb4e576c6a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:14:44 -0500 Subject: [PATCH 08/17] fix(caching): keep embedding cache hits aligned with request inputs (#42571) * fix(caching): keep embedding cache hits aligned with request inputs Partial hits now send only the uncached inputs to the provider and merge fresh vectors back into their original positions. Responses whose item count differs from the input count (one input scoring many documents) are no longer written to the per-input cache, since a later hit would return a single item. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): drop mutable collection builds flagged by the type discipline gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): bypass embedding cache entries written before the per input cardinality check Embedding cache entries now carry format_version and readers treat entries without it as misses, so entries that only hold the first row of a multi row response are refetched instead of served until their TTL expires. The provider call also receives a copy of the request kwargs with the uncached inputs rather than mutating the caller's mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): assert a partial embedding cache hit becomes a full hit on repeat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): await pending embedding cache writes before asserting on cache hits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): validate cached embeddings without mutating responses or request kwargs Validate cache rows through a frozen pydantic model so import does not depend on TypeAdapter support for ReadOnly TypedDicts, accept string embeddings, build the merged partial hit response instead of mutating the cached one, and hand the provider request mapping to post call hooks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): keep cache_hit and response_ms on merged partial embedding hits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 55 ++++----- litellm/caching/caching_handler.py | 89 +++++++++----- litellm/types/caching.py | 18 +-- litellm/utils.py | 16 ++- tests/test_litellm/caching/test_caching.py | 113 +++++++++++++++++- .../caching/test_caching_handler.py | 45 ++++++- 6 files changed, 264 insertions(+), 72 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 6f99eb616b0..a31cad4af29 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -781,35 +781,23 @@ class Cache: Convert any embedding response into the standardized CachedEmbedding TypedDict format. """ try: - if isinstance(embedding_response, dict): - return { - "embedding": embedding_response.get("embedding"), - "index": embedding_response.get("index"), - "object": embedding_response.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - elif hasattr(embedding_response, "model_dump"): - data = embedding_response.model_dump() - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - else: - data = vars(embedding_response) - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } + data: Final = ( + embedding_response + if isinstance(embedding_response, dict) + else embedding_response.model_dump() + if hasattr(embedding_response, "model_dump") + else vars(embedding_response) + ) + cached: Final[CachedEmbedding] = { + "embedding": data.get("embedding"), + "index": data.get("index"), + "object": data.get("object"), + "model": model, + "prompt_tokens": prompt_tokens, + "prompt_tokens_details": prompt_tokens_details, + "format_version": EMBEDDING_CACHE_FORMAT_VERSION, + } + return cached except KeyError as e: raise ValueError(f"Missing expected key in embedding response: {e}") @@ -925,6 +913,15 @@ class Cache: if self.should_use_cache(**kwargs) is not True: return + input_count: Final = len(kwargs["input"]) if isinstance(kwargs["input"], list) else 1 + if len(result.data) != input_count: + verbose_logger.debug( + "LiteLLM Cache: skipping embedding cache write, %d inputs but %d embeddings in the response", + input_count, + len(result.data), + ) + return + # set default ttl if not set if self.ttl is not None: kwargs["ttl"] = self.ttl diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 36c3b744a06..4afd0e7caaa 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -21,7 +21,7 @@ import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm._logging import print_verbose, verbose_logger @@ -34,7 +34,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( from litellm.litellm_core_utils.logging_utils import ( _assemble_complete_response_from_streaming_chunks, ) -from litellm.types.caching import CachedEmbedding +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.rerank import RerankResponse @@ -77,6 +77,7 @@ class CachingHandlerResponse(BaseModel): cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call + embedding_uncached_input: list[str | list[int]] | None = None in_memory_cache_obj: Final = InMemoryCache() @@ -168,6 +169,37 @@ def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None: return request_kwargs.get("cache_key", None) +class _CachedEmbeddingRecord(BaseModel): + model_config = ConfigDict(frozen=True) + + embedding: list[float] | str | None + index: int | None + object: str | None + model: str | None + prompt_tokens: int | None + prompt_tokens_details: dict | None + format_version: int + + +def _current_format_embedding_entry(entry: object) -> CachedEmbedding | None: + try: + record: Final = _CachedEmbeddingRecord.model_validate(entry) + except ValidationError: + return None + if record.format_version != EMBEDDING_CACHE_FORMAT_VERSION: + return None + cached: Final[CachedEmbedding] = { + "embedding": record.embedding, + "index": record.index, + "object": record.object, + "model": record.model, + "prompt_tokens": record.prompt_tokens, + "prompt_tokens_details": record.prompt_tokens_details, + "format_version": record.format_version, + } + return cached + + class LLMCachingHandler: def __init__( self, @@ -320,6 +352,7 @@ class LLMCachingHandler: return CachingHandlerResponse( final_embedding_cached_response=final_embedding_cached_response, embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, + embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), ) verbose_logger.debug("CACHE RESULT: %s", cached_result) @@ -657,32 +690,30 @@ class LLMCachingHandler: if _caching_handler_response.final_embedding_cached_response is None: return embedding_response - idx = 0 - final_data_list: Final = [] - for item in _caching_handler_response.final_embedding_cached_response.data: - if item is None and embedding_response.data is not None: - final_data_list.append(embedding_response.data[idx]) - idx += 1 - else: - final_data_list.append(item) - - _caching_handler_response.final_embedding_cached_response.data = final_data_list - _caching_handler_response.final_embedding_cached_response._hidden_params["cache_hit"] = True - _caching_handler_response.final_embedding_cached_response._response_ms = ( - end_time - start_time - ).total_seconds() * 1000 - - ## USAGE - if ( - _caching_handler_response.final_embedding_cached_response.usage is not None - and embedding_response.usage is not None - ): - _caching_handler_response.final_embedding_cached_response.usage = self.combine_usage( - usage1=_caching_handler_response.final_embedding_cached_response.usage, - usage2=embedding_response.usage, - ) - - return _caching_handler_response.final_embedding_cached_response + cached: Final = _caching_handler_response.final_embedding_cached_response + fresh_items: Final = iter(embedding_response.data or ()) + merged_usage: Final = ( + self.combine_usage(usage1=cached.usage, usage2=embedding_response.usage) + if cached.usage is not None and embedding_response.usage is not None + else cached.usage + ) + merged: Final = EmbeddingResponse( + model=cached.model, + data=[ # mutable-ok: EmbeddingResponse.data is a pydantic list field + item + if item is not None + else Embedding(embedding=next(fresh_items)["embedding"], index=position, object="embedding") + for position, item in enumerate(cached.data) + ], + usage=merged_usage, + hidden_params={ # mutable-ok: EmbeddingResponse._hidden_params is a mutable dict field + **cached._hidden_params, + "cache_hit": True, + }, + _response_headers=cached._response_headers, + ) + merged._response_ms = (end_time - start_time).total_seconds() * 1000 + return merged def _async_log_cache_hit_on_callbacks( self, @@ -770,7 +801,7 @@ class LLMCachingHandler: dynamic_cache_object=self.dual_cache, ) ) - cached_result = await asyncio.gather(*tasks) + cached_result = [_current_format_embedding_entry(entry) for entry in await asyncio.gather(*tasks)] ## check if cached result is None ## if cached_result is not None and isinstance(cached_result, list): # set cached_result to None if all elements are None diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 747f202d35e..6d42556a9c4 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Final, Literal, Optional, Union from pydantic import BaseModel -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class LiteLLMCacheType(str, Enum): @@ -137,12 +137,16 @@ class HealthCheckCacheParams(BaseModel): redis_version: str | int | float | None = None +EMBEDDING_CACHE_FORMAT_VERSION: Final = 2 + + class CachedEmbedding(TypedDict): """Type definition for cached embedding objects""" - embedding: list[float] | None - index: int | None - object: str | None - model: str | None - prompt_tokens: int | None - prompt_tokens_details: dict | None + embedding: ReadOnly[list[float] | str | None] + index: ReadOnly[int | None] + object: ReadOnly[str | None] + model: ReadOnly[str | None] + prompt_tokens: ReadOnly[int | None] + prompt_tokens_details: ReadOnly[dict | None] + format_version: ReadOnly[int] diff --git a/litellm/utils.py b/litellm/utils.py index b9d56b25653..b5d396030f7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2015,13 +2015,19 @@ def client(original_function): print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL + call_kwargs: Final = ( + {**kwargs, "input": _caching_handler_response.embedding_uncached_input} + if _caching_handler_response is not None + and _caching_handler_response.embedding_uncached_input is not None + else kwargs + ) try: - result = await original_function(*args, **kwargs) + result = await original_function(*args, **call_kwargs) except Exception as deployment_error: _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with try: await async_post_call_failure_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, exception=deployment_error, call_type=call_type, ) @@ -2062,7 +2068,7 @@ def client(original_function): post_call_processing( original_response=result, model=model, - optional_params=kwargs, + optional_params=call_kwargs, original_function=original_function, rules_obj=rules_obj, ) @@ -2070,7 +2076,7 @@ def client(original_function): _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: result = await async_post_call_success_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, response=result, call_type=_call_type_enum, ) @@ -2079,7 +2085,7 @@ def client(original_function): await _llm_caching_handler.async_set_cache( result=result, original_function=original_function, - kwargs=kwargs, + kwargs=call_kwargs, args=args, ) diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index c7ec8abc31e..2e4122530d8 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -1,3 +1,4 @@ +import asyncio import logging import re from unittest.mock import MagicMock @@ -6,8 +7,9 @@ import pytest import litellm.caching.redis_cache as redis_cache_module from litellm.caching.caching import Cache +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.caching.redis_cache import RedisCache, _RedisTimeoutLogThrottle -from litellm.types.caching import LiteLLMCacheType, SemanticCacheScope +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, LiteLLMCacheType, SemanticCacheScope from litellm.types.utils import Embedding, EmbeddingResponse, Usage @@ -278,3 +280,112 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key( model="claude-sonnet-4-5", messages=messages, **anthropic_param ) + + +@pytest.mark.asyncio +async def test_embedding_cache_skips_write_when_one_input_yields_many_embeddings(monkeypatch): + """A cross-encoder behind /embeddings returns one score per document for a single + input string; caching data[0] per input would make the second call return 1 score.""" + import litellm + from litellm import CustomLLM + + class ScoreEveryDocument(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding=[float(i)], index=i, object="embedding") for i in range(5)], + ) + + scorer = ScoreEveryDocument() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "score-every-doc", "custom_handler": scorer}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "score-every-doc"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "score-every-doc"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + batch = '{"query": "q", "documents": ["a", "b", "c", "d", "e"]}' + first = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + + assert scorer.provider_calls == 2 + assert [len(first.data), len(second.data)] == [5, 5] + + +@pytest.mark.asyncio +async def test_embedding_cache_refetches_entries_written_without_format_version(monkeypatch): + import litellm + from litellm import CustomLLM + + class EmbedLength(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = EmbedLength() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-length", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-length"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-length"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + await litellm.aembedding(model="embed-length/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + store = litellm.cache.cache.cache_dict + stored = [entry["response"] for entry in store.values()] + assert [entry["format_version"] for entry in stored] == [EMBEDDING_CACHE_FORMAT_VERSION], stored + legacy_store = { + key: { + **entry, + "response": { + field: value + for field, value in {**entry["response"], "embedding": [-1.0]}.items() + if field != "format_version" + }, + } + for key, entry in store.items() + } + monkeypatch.setattr(litellm.cache.cache, "cache_dict", legacy_store) + + refetched = await litellm.aembedding(model="embed-length/m", input=["abcd"]) + + assert embedder.provider_calls == 2, "an entry written without format_version must be a cache miss" + assert [item["embedding"] for item in refetched.data] == [[4.0]] + + +@pytest.mark.asyncio +async def test_embedding_cache_serves_base64_string_embeddings_on_repeat(monkeypatch): + import litellm + from litellm import CustomLLM + + class Base64Embedder(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding="AACAPwAAAEA=", index=idx, object="embedding") for idx, _ in enumerate(input)], + ) + + embedder = Base64Embedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-b64", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-b64"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-b64"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + first = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + + assert embedder.provider_calls == 1, "a string embedding written to the cache must be served on repeat" + assert [item["embedding"] for item in second.data] == [item["embedding"] for item in first.data] == ["AACAPwAAAEA="] diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 39018dca41d..6956a6932d5 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -11,7 +11,7 @@ from fastapi.testclient import TestClient from datetime import datetime from unittest.mock import AsyncMock -from litellm.caching.caching_handler import LLMCachingHandler +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES, LLMCachingHandler @pytest.mark.asyncio @@ -780,3 +780,46 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl assert hit.cached_result.choices[0].message.content == "done" logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once() assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True + + +@pytest.mark.asyncio +async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_order(monkeypatch): + import litellm + from litellm import CustomLLM + from litellm.caching.caching import Cache + from litellm.types.utils import Embedding, EmbeddingResponse + + class RecordingEmbedder(CustomLLM): + provider_inputs: tuple[tuple[str, ...], ...] = () + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_inputs = (*self.provider_inputs, tuple(input)) + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = RecordingEmbedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "recording-embedder", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "recording-embedder"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "recording-embedder"]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + + await litellm.aembedding(model="recording-embedder/m", input=["aa", "bbbb"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + mixed_input = ["c", "aa", "ddd", "bbbb", "eeeee"] + response = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert embedder.provider_inputs == (("aa", "bbbb"), ("c", "ddd", "eeeee")), embedder.provider_inputs + assert [item["index"] for item in response.data] == [0, 1, 2, 3, 4] + assert [item["embedding"] for item in response.data] == [[float(len(text))] for text in mixed_input] + assert response._hidden_params["cache_hit"] is True, "a partial hit must still be reported as a cache hit" + + repeat = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + + assert len(embedder.provider_inputs) == 2, embedder.provider_inputs + assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input] From 95d4613bd3f64cbbdb752f77c935015166718409 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:14:48 -0700 Subject: [PATCH 09/17] chore(prices): sync xAI prices: 3 models, 3 new [3 with gaps] (#42591) * chore(prices): sync xAI prices: 3 models, 3 new [3 with gaps] xai/grok-code-fast: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema xai/grok-code-fast-1: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema xai/grok-code-fast-1-0825: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema * fix(prices): add context limits and reasoning flag to xai grok-code-fast aliases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prices): keep the xai sync diff limited to the alias fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 63 +++++++++++++++++++ model_prices_and_context_window.json | 63 +++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3f1d26e9adc..224129fbd25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72335,5 +72335,68 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3f1d26e9adc..224129fbd25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72335,5 +72335,68 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 238f4341532ae6dd150de55e9302c08a8e031c66 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:16:46 -0700 Subject: [PATCH 10/17] fix(otel): record the GenAI exception event through the Logs API on both OpenTelemetry lines (#42431) * fix(otel): record the GenAI exception event without the removed Events API OpenTelemetry removed opentelemetry._events in 1.44.0, so the three imports of it broke 7 modules under litellm.integrations.otel, including the entry point. Two things then failed quietly: with LITELLM_OTEL_V2 set the otel callback resolved to None and nothing was exported, and with it unset the newrelic callback was dropped as well, because that branch imports the v2 logger ungated Build and emit the event through the Logs API, which both lines carry. The event name keeps riding the event.name attribute: the event_name log record field that replaces it only exists from 1.44.0, and this package pins 1.28.0, so the attribute is the only form both can write. It is also what the Events API wrote, so exported events keep their shape Emitting a plain record drops the default the Events SDK applied, so the timestamp now falls back to time_ns() here * style(otel): trim the event name key and regression test prose Keep only the constraint a reader cannot infer from the code, that the event_name record field does not exist on the pinned OpenTelemetry line * fix(otel): export the GenAI exception event on both OpenTelemetry 1.28 and 1.44 lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the record selection comment Co-authored-by: Pawan-Shahane Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): collapse the record selection conditional for ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): build the record fields with a dict literal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): set the native event_name on the 1.44 log record Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): spell out the record kwargs so the type gate sees each call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): suppress the version-gated kwargs for the type gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the version-window prose and correct the event_name suppression reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): wrap the compat test docstring to the line limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Pawan-Shahane Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/logger.py | 2 +- litellm/integrations/otel/model/semconv.py | 1 + litellm/integrations/otel/plumbing/events.py | 54 +++++--- .../integrations/otel/plumbing/providers.py | 8 +- .../otel/test_otel_v2_components.py | 125 ++++++++++++++++++ 5 files changed, 168 insertions(+), 22 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6b673967427..20cdf9662e1 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -240,7 +240,7 @@ class OpenTelemetryV2(CustomLogger): provider: Final = resolve_logger_provider(self.config, logger_provider) if provider is None: return None - return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) # ====================================================================== # # Proxy global registration diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index d3628005bac..f552ba37655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -245,6 +245,7 @@ class GenAIEvent: details, unlike the deprecated ``error.message`` span attribute. """ + NAME_KEY: Final = "event.name" OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py index e7b8e22ddcd..e95d31f886f 100644 --- a/litellm/integrations/otel/plumbing/events.py +++ b/litellm/integrations/otel/plumbing/events.py @@ -9,18 +9,28 @@ emitting that event; the exporter pipeline it rides is built in """ from dataclasses import dataclass +from time import time_ns from typing import Final -from opentelemetry._events import Event, EventLogger +from opentelemetry._logs import Logger, LogRecord from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk.resources import Resource from opentelemetry.trace import SpanContext from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent +try: + from opentelemetry.sdk._logs import LogRecord as _SDKLogRecord +except ImportError: + _SDKLogRecord = None + +SDK_LOG_RECORD: Final[type[LogRecord] | None] = _SDKLogRecord + @dataclass(frozen=True, slots=True) class GenAIEventRecorder: - event_logger: EventLogger + event_logger: Logger + resource: Resource | None = None def record_operation_exception( self, @@ -30,24 +40,36 @@ class GenAIEventRecorder: stack_trace: str | None, timestamp_ns: int | None, ) -> None: - # ``exception.type`` and ``exception.message`` are the semconv-required - # pair and always ride the event; only the recommended stacktrace is - # conditional on the payload carrying one. stacktrace: Final = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () - self.event_logger.emit( - Event( - name=GenAIEvent.OPERATION_EXCEPTION, - timestamp=timestamp_ns, + attributes: Final = dict( + ( + (GenAIEvent.NAME_KEY, GenAIEvent.OPERATION_EXCEPTION), + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ) + record: Final[LogRecord] = ( + SDK_LOG_RECORD( + timestamp=timestamp_ns or time_ns(), trace_id=span_context.trace_id, span_id=span_context.span_id, trace_flags=span_context.trace_flags, severity_number=SeverityNumber.WARN, - attributes=dict( - ( - (ExceptionEvent.TYPE, error_type), - (ExceptionEvent.MESSAGE, message), - *stacktrace, - ) - ), + body=message, + attributes=attributes, + resource=self.resource, # pyright: ignore[reportCallIssue] # SDK-only kwarg absent from the API LogRecord signature on the pin + ) + if SDK_LOG_RECORD is not None + else LogRecord( + timestamp=timestamp_ns or time_ns(), + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + body=message, + attributes=attributes, + event_name=GenAIEvent.OPERATION_EXCEPTION, # pyright: ignore[reportCallIssue] # kwarg exists only on OTel 1.38+, absent from the pinned API signature ) ) + self.event_logger.emit(record) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index d52736a1303..e3474edaf14 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -9,11 +9,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics, trace -from opentelemetry._events import EventLogger -from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider +from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.sdk._events import EventLoggerProvider from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider from opentelemetry.sdk._logs.export import ( BatchLogRecordProcessor, @@ -1042,8 +1040,8 @@ def resolve_logger_provider( return provider -def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: - return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> Logger: + return provider.get_logger(name, litellm_version) def build_meter_provider( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 79747ac9956..07705e17d9a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -1290,6 +1290,50 @@ def test_operation_exception_log_event_always_carries_required_pair(): assert ExceptionEvent.STACKTRACE not in attributes +def test_operation_exception_log_event_records_without_the_events_api(): + """Recording must not import the Events API modules (removed upstream in 1.44.0); + the SDK record path still exports.""" + import importlib + import sys + from unittest.mock import patch + + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + plumbing = ("litellm.integrations.otel.plumbing.events", "litellm.integrations.otel.plumbing.providers") + without_events_api = { + **{name: module for name, module in sys.modules.items() if name not in plumbing}, + "opentelemetry._events": None, + "opentelemetry.sdk._events": None, + } + with patch.dict(sys.modules, without_events_api, clear=True): + events_mod = importlib.import_module(plumbing[0]) + providers_mod = importlib.import_module(plumbing[1]) + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers_mod.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = events_mod.GenAIEventRecorder(providers_mod.get_event_logger(logger_provider)) + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.severity_number == SeverityNumber.WARN + assert record.timestamp is not None + + def test_operation_exception_log_event_not_emitted_on_success(): engine, span_exporter, log_exporter = _engine_with_event_recorder() engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) @@ -1425,6 +1469,87 @@ def test_genai_mapper_guardrail_cost_in_spend_attr(): assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed)) +def _sampled_span_context(): + from opentelemetry.trace import SpanContext, TraceFlags, TraceState + + return SpanContext( + trace_id=0x0AF7651916CD43DD8448EB211C80319C, + span_id=0x00F067AA0BA902B7, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + ) + + +def test_operation_exception_log_event_exports_through_console_exporter(): + """The emitted record serializes through a real SDK exporter: the console + exporter only handles SDK-shaped records (``to_json`` plus a resource), so + an API-shaped record crashed the export under the repo's pinned OTel.""" + import io + import json as json_mod + + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import ConsoleLogExporter, SimpleLogRecordProcessor + from opentelemetry.sdk.resources import Resource + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + out = io.StringIO() + logger_provider = LoggerProvider(resource=Resource.create({"service.name": "otel-event-test"})) + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(ConsoleLogExporter(out=out))) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + exported = json_mod.loads(out.getvalue()) + assert exported["attributes"][GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert exported["attributes"][ExceptionEvent.TYPE] == "RateLimitError" + assert exported["attributes"][ExceptionEvent.MESSAGE] == "rate limited" + assert exported["body"] == "rate limited" + assert exported["resource"]["attributes"]["service.name"] == "otel-event-test" + + +def test_operation_exception_log_event_encodes_for_otlp(): + """The OTLP log encoder reads ``log_record.resource`` and rejects a None + body on the pinned OTel line, so the event must encode into a real + ExportLogsServiceRequest, not only land in an in-memory exporter.""" + from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.model.semconv import GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + request = encode_logs(log_exporter.get_finished_logs()) + (resource_logs,) = request.resource_logs + (scope_logs,) = resource_logs.scope_logs + (encoded,) = scope_logs.log_records + encoded_attrs = {a.key: a.value.string_value for a in encoded.attributes} + assert encoded_attrs[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert encoded.body.string_value == "rate limited" + resource_attrs = {a.key: a.value.string_value for a in resource_logs.resource.attributes} + assert resource_attrs["service.name"] == logger_provider.resource.attributes["service.name"] + + + + def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: for key in ( "SSL_VERIFY", From 3db94b932ec792c101149acafc3a1ff2e3004456 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:44:00 -0700 Subject: [PATCH 11/17] fix(spend): return 400 from /spend/calculate for a model with no pricing row (#42497) * fix(spend): return 400 from /spend/calculate for a model with no pricing row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): assert error type and param for unpriced /spend/calculate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): move the repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: alias ModelNotMappedError re-export to satisfy F401 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(utils): raise ModelNotMappedError only when the pricing row is missing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/exceptions.py | 4 ++++ .../spend_management_endpoints.py | 7 +++++++ litellm/utils.py | 18 +++++++++++------ tests/integration/contracts.json | 3 +++ .../integration/spend/test_spend_calculate.py | 20 +++++++++++++++++++ .../test_spend_management_endpoints.py | 17 ++++++++++++++++ tests/test_litellm/test_utils.py | 7 +++++++ 8 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 tests/integration/spend/test_spend_calculate.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 471b273f00d..c8df4394a06 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1402,6 +1402,7 @@ from .exceptions import ( JSONSchemaValidationError, LITELLM_EXCEPTION_TYPES, MockException, + ModelNotMappedError as ModelNotMappedError, ) from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server diff --git a/litellm/exceptions.py b/litellm/exceptions.py index c8de2ab12ed..3bae8a95ef6 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -991,6 +991,10 @@ LITELLM_EXCEPTION_TYPES: Final = [ ] +class ModelNotMappedError(Exception): + pass + + class BudgetExceededError(Exception): def __init__( self, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index b5980f9b224..822b827f985 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2323,6 +2323,13 @@ async def calculate_spend(request: SpendCalculateRequest): param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) + if isinstance(e, litellm.exceptions.ModelNotMappedError): + raise ProxyException( + message=str(e), + type="invalid_request_error", + param="model", + code=status.HTTP_400_BAD_REQUEST, + ) error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), diff --git a/litellm/utils.py b/litellm/utils.py index b5d396030f7..01f6fee3594 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -472,6 +472,7 @@ from .exceptions import ( BudgetExceededError, ContentPolicyViolationError, ContextWindowExceededError, + ModelNotMappedError, NotFoundError, OpenAIError, PermissionDeniedError, @@ -5830,6 +5831,13 @@ def _is_potential_model_name_in_model_cost( _ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN +def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str: + return ( + f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. " + "Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." + ) + + def _get_model_info_helper( model: str, custom_llm_provider: str | None = None, @@ -6012,9 +6020,7 @@ def _get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: - raise ValueError( - "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" - ) + raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: # default value to 0, be noisy about this @@ -6249,11 +6255,11 @@ def _get_model_info_helper( if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: returned_model_info[cost_key] = cost_value return returned_model_info + except ModelNotMappedError: + raise except Exception as e: verbose_logger.debug("Error getting model info: %s", e) - raise Exception( - f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." - ) + raise Exception(_model_not_mapped_message(model, custom_llm_provider)) def _build_model_info( diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index cdf534bb5a1..1ae760467f1 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -275,6 +275,9 @@ "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" ], + "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ + "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" + ], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ "mgmt.key.update.project_detach_denied_to_restricted_actor" ], diff --git a/tests/integration/spend/test_spend_calculate.py b/tests/integration/spend/test_spend_calculate.py new file mode 100644 index 00000000000..855dd2b21f3 --- /dev/null +++ b/tests/integration/spend/test_spend_calculate.py @@ -0,0 +1,20 @@ +import uuid +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, object_value, string_value + + +@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.rejects_unpriced_model") +def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> None: + model: Final = f"openrouter/integration-unpriced-{uuid.uuid4().hex}" + response: Final = gateway.request( + "POST", + "/spend/calculate", + {"model": model, "messages": [{"role": "user", "content": "price this request"}]}, + ) + assert response.status_code == 400, response.text + error: Final = object_value(JSON_OBJECT.validate_json(response.text)["error"]) + assert error["type"] == "invalid_request_error", response.text + assert error["param"] == "model", response.text + assert model in string_value(error["message"]), response.text diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index e41c027d962..c6a4173b583 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -257,6 +257,8 @@ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.proxy._types import ( LitellmUserRoles, Member, + ProxyException, + SpendCalculateRequest, SpendLogsPayload, UserAPIKeyAuth, ) @@ -7835,3 +7837,18 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo assert custom_logger.requested_ids == [] finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_calculate_spend_unpriced_model_returns_400(): + model = "openrouter/unit-test-unpriced-model" + with patch("litellm.proxy.proxy_server.llm_router", None): + with pytest.raises(ProxyException) as exc_info: + await spend_management_endpoints.calculate_spend( + SpendCalculateRequest(model=model, messages=[{"role": "user", "content": "hi"}]) + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "model" + assert model in exc_info.value.message diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 07982f51153..79462d16a8c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -230,6 +230,13 @@ def test_get_model_info_prefers_exact_dated_key_over_stripped( assert info["key"] == expected_key +def test_get_model_info_internal_failure_is_not_reported_as_unmapped() -> None: + with patch("litellm.utils._get_potential_model_names", side_effect=RuntimeError("malformed metadata")): + with pytest.raises(Exception, match="This model isn't mapped yet") as exc_info: + litellm.utils._get_model_info_helper(model="gpt-4o", custom_llm_provider="openai") + assert not isinstance(exc_info.value, litellm.ModelNotMappedError) + + def test_check_provider_match_azure_ai_allows_openai_and_azure(): """ Test that azure_ai provider can match openai and azure models. From 58a05a9eae5b31d2a71d1766aa5b5a153d8c2263 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:44:30 -0700 Subject: [PATCH 12/17] fix(anthropic): return 400 instead of 500 when a content list holds a bare string (#42420) * fix(anthropic): skip non-dict content items in beta-header and file-id helpers so malformed content lists return 400 instead of 500 Fixes #42094 Supersedes #42101 Co-authored-by: Pawan-Shahane Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): spawn the DB-less regression proxy with -P so the cwd cannot shadow the pinned checkout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): launch the DB-less proxy via -I -c with an explicit sys.path so python 3.10 works, drop DIRECT_URL, remove restating docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): gate the self-booted DB-less proxy behind the owned_gateway opt-in the Buildkite container cannot satisfy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): move the bare string content item repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): wrap the anthropic bare string wire test to the 120 column limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): wrap anthropic common_utils test literals to the 120 column limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: fix ruff findings in touched test files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Pawan-Shahane Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 2 +- litellm/llms/anthropic/common_utils.py | 4 +- tests/integration/contracts.json | 6 ++ .../providers/test_anthropic_wire.py | 24 ++++++- ...ore_utils_prompt_templates_common_utils.py | 15 +++- .../anthropic/test_anthropic_common_utils.py | 70 +++++++++++++++++++ 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 1e96e20a03b..6c45622649f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1657,7 +1657,7 @@ def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: if isinstance(content, str): continue for c in content: - if c["type"] == "file": + if isinstance(c, dict) and c["type"] == "file": file_object = cast(ChatCompletionFileObject, c) file_object_file_field = file_object.get("file") if not isinstance(file_object_file_field, dict): diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0c3c8996789..c0e6006633e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -314,7 +314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): _message_content = message.get("content") if _message_content is not None and isinstance(_message_content, list): for content in _message_content: - if "cache_control" in content: + if isinstance(content, dict) and "cache_control" in content: return True return False @@ -359,7 +359,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): for message in messages: if "content" in message and message["content"] is not None and isinstance(message["content"], list): for content in message["content"]: - if "type" in content and content["type"] != "text": + if isinstance(content, dict) and "type" in content and content["type"] != "text": return True return False diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 1ae760467f1..8c4ce4debbc 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -159,6 +159,12 @@ "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": [ "other.routing.redis.owned_outage_recovers_serving_and_response_cache" ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": [ "other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates" diff --git a/tests/integration/providers/test_anthropic_wire.py b/tests/integration/providers/test_anthropic_wire.py index 64160fa85aa..27895440712 100644 --- a/tests/integration/providers/test_anthropic_wire.py +++ b/tests/integration/providers/test_anthropic_wire.py @@ -3,7 +3,6 @@ import uuid from typing import Final import pytest - from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server @@ -59,3 +58,26 @@ def test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contra parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) assert parsed["cost_breakdown"]["input_cost"] == pytest.approx(0.0245) assert parsed["cost_breakdown"]["output_cost"] == pytest.approx(0.008) + + +@pytest.mark.covers("other.provider_wire.anthropic.bare_string_content_item_is_client_error") +@pytest.mark.parametrize( + "text", [pytest.param("what type of file is this?", id="type_word"), pytest.param("hello", id="plain")] +) +def test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire( + gateway: Gateway, text: str +) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"upstream must not be reached: {request.target}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-5-20250929", api_base=wire.url, api_key="synthetic-anthropic-key" + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 16, "timeout": 5, "messages": [{"role": "system", "content": [text]}]}, + ) + assert response.status_code == 400, response.text + assert wire.drain() == () diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 62cf5680266..79c50bf2369 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -4,7 +4,6 @@ import json import os import sys from typing import Final -from unittest.mock import MagicMock, patch import pytest @@ -356,6 +355,20 @@ def test_get_file_ids_from_messages_file_field_not_dict(): assert get_file_ids_from_messages(messages) == [] +def test_get_file_ids_from_messages_skips_bare_string_content_items(): + messages = [ + { + "role": "user", + "content": [ + "what type of file is this?", + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): """`update_messages_with_model_file_ids` is also called on user content before provider dispatch. It must tolerate non-OpenAI file blocks the same diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 945033c5cac..1a21b6d4394 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -14,6 +14,7 @@ import json import os import sys from types import SimpleNamespace +from typing import Final from unittest.mock import patch import pytest @@ -2275,3 +2276,72 @@ def test_create_anthropic_model_list_response_lists_ids_as_told(): assert (gpt["id"], gpt["display_name"], gpt["max_input_tokens"]) == ("claude-router-gpt-4o[1m]", "GPT 4o", 1000000) assert (haiku["id"], haiku["display_name"]) == ("claude-haiku-4-5", "claude-haiku-4-5") assert (response["first_id"], response["last_id"]) == ("claude-router-gpt-4o[1m]", "claude-haiku-4-5") + + +class TestMalformedContentListItems: + @pytest.mark.parametrize( + "content", + [ + pytest.param(["what type of file is this?"], id="string_containing_type"), + pytest.param(["how do I set cache_control?"], id="string_containing_cache_control"), + pytest.param([None], id="none_item"), + pytest.param([5], id="int_item"), + pytest.param([["nested"]], id="list_item"), + ], + ) + def test_beta_headers_resolve_for_non_dict_content_items(self, content: list[object]) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + messages: Final = [{"role": "user", "content": content}] + + headers: Final = config.validate_environment( + headers={}, + model="claude-sonnet-4-5", + messages=messages, + optional_params={}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert config.is_cache_control_set(messages) is False + assert config.is_pdf_used(messages) is False + + def test_real_content_parts_still_set_their_beta_headers(self) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + + assert config.is_pdf_used([{"role": "user", "content": [{"type": "image", "source": {}}]}]) is True + assert config.is_pdf_used([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]) is False + assert ( + config.is_cache_control_set( + [ + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}], + } + ] + ) + is True + ) + + def test_mixed_list_keeps_detecting_the_valid_part(self) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + messages: Final = [{"role": "user", "content": ["what type of file is this?", {"type": "image", "source": {}}]}] + + assert config.is_pdf_used(messages) is True + + def test_litellm_completion_rejects_bare_string_content_item_as_bad_request(self) -> None: + import litellm + + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model="anthropic/claude-haiku-4-5-20251001", + messages=[{"role": "user", "content": ["what type of file is this?"]}], + api_key=FAKE_REGULAR_KEY, + max_tokens=5, + ) From 7177d3b6d11dc208e2531f05df4df7a1f33b228a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:50:28 +0000 Subject: [PATCH 13/17] feat(rust): add standalone cost calculator (#42604) * feat: add standalone Rust text pricing crate * feat(rust): harden standalone cost calculator --------- Co-authored-by: Yujong Lee --- litellm-rust/Cargo.lock | 8 + litellm-rust/crates/cost/Cargo.toml | 14 + litellm-rust/crates/cost/README.md | 13 + litellm-rust/crates/cost/benches/calculate.rs | 66 +++ litellm-rust/crates/cost/examples/charge.rs | 40 ++ litellm-rust/crates/cost/src/lib.rs | 405 ++++++++++++++++ litellm-rust/crates/cost/tests/calculation.rs | 458 ++++++++++++++++++ .../cost/tests/generate_python_reference.py | 96 ++++ .../crates/cost/tests/python_reference.tsv | 9 + 9 files changed, 1109 insertions(+) create mode 100644 litellm-rust/crates/cost/Cargo.toml create mode 100644 litellm-rust/crates/cost/README.md create mode 100644 litellm-rust/crates/cost/benches/calculate.rs create mode 100644 litellm-rust/crates/cost/examples/charge.rs create mode 100644 litellm-rust/crates/cost/src/lib.rs create mode 100644 litellm-rust/crates/cost/tests/calculation.rs create mode 100644 litellm-rust/crates/cost/tests/generate_python_reference.py create mode 100644 litellm-rust/crates/cost/tests/python_reference.tsv diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d9425fc6bd7..f3510000c44 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2956,6 +2956,14 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-cost" +version = "0.1.0" +dependencies = [ + "criterion", + "proptest", +] + [[package]] name = "litellm-framing" version = "0.1.0" diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml new file mode 100644 index 00000000000..b85f8a8f876 --- /dev/null +++ b/litellm-rust/crates/cost/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-cost" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dev-dependencies] +criterion.workspace = true +proptest.workspace = true + +[[bench]] +name = "calculate" +harness = false diff --git a/litellm-rust/crates/cost/README.md b/litellm-rust/crates/cost/README.md new file mode 100644 index 00000000000..ee4722b6438 --- /dev/null +++ b/litellm-rust/crates/cost/README.md @@ -0,0 +1,13 @@ +# litellm-cost + +This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration + +Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values + +The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates + +`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract + +The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes + +This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage diff --git a/litellm-rust/crates/cost/benches/calculate.rs b/litellm-rust/crates/cost/benches/calculate.rs new file mode 100644 index 00000000000..2561b606e6c --- /dev/null +++ b/litellm-rust/crates/cost/benches/calculate.rs @@ -0,0 +1,66 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use litellm_cost::{ + Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates, + Usage, calculate, compile, +}; +use std::hint::black_box; + +fn bench(c: &mut Criterion) { + let pricing = Pricing { + standard: Rates { + input: Rate::Value(0.000002), + output: Rate::Value(0.000008), + cache_read: Rate::Value(0.0000005), + cache_write: Rate::Missing, + cache_write_1h: Rate::Missing, + }, + tiers: &[], + thresholds: &[], + off_peak: None, + }; + let request = Request { + usage: Usage { + prompt_tokens: 1000, + completion_tokens: 200, + cache_read_tokens: 250, + cache_write_tokens: 0, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + }; + let plan = compile(&pricing).unwrap(); + c.bench_function("native_compiled_calculation", |b| { + b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap())) + }); + c.bench_function("native_full_wrapper", |b| { + b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap())) + }); + c.bench_function("native_rate_compilation", |b| { + b.iter(|| black_box(compile(black_box(&pricing)).unwrap())) + }); + let threshold = ThresholdRates { + above_prompt_tokens: 1000, + standard: Rates { + input: Rate::Value(0.000004), + output: Rate::Value(0.000016), + ..Rates::EMPTY + }, + tiers: &[], + }; + let threshold_pricing = Pricing { + thresholds: &[threshold], + ..pricing + }; + let threshold_plan = compile(&threshold_pricing).unwrap(); + c.bench_function("native_threshold_boundary", |b| { + b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap())) + }); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/litellm-rust/crates/cost/examples/charge.rs b/litellm-rust/crates/cost/examples/charge.rs new file mode 100644 index 00000000000..a3a83febd74 --- /dev/null +++ b/litellm-rust/crates/cost/examples/charge.rs @@ -0,0 +1,40 @@ +use litellm_cost::{ + Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile, +}; + +fn main() { + let pricing = Pricing { + standard: Rates { + input: Rate::Value(2.0), + output: Rate::Value(4.0), + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + cache_write_1h: Rate::Missing, + }, + tiers: &[], + thresholds: &[], + off_peak: None, + }; + let request = Request { + usage: Usage { + prompt_tokens: 100, + completion_tokens: 20, + cache_read_tokens: 25, + cache_write_tokens: 10, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + }; + let cost = compile(&pricing).unwrap().calculate(&request).unwrap(); + println!( + "input={} output={} total={}", + cost.input(), + cost.output(), + cost.total() + ); +} diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs new file mode 100644 index 00000000000..32cc755ce15 --- /dev/null +++ b/litellm-rust/crates/cost/src/lib.rs @@ -0,0 +1,405 @@ +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Rate { + Missing, + Null, + Value(f64), +} + +impl Rate { + fn value(self) -> Option { + match self { + Self::Value(value) => Some(value), + Self::Missing | Self::Null => None, + } + } + + fn or(self, fallback: Self) -> Self { + if self.value().is_some() { + self + } else { + fallback + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Rates { + pub input: Rate, + pub output: Rate, + pub cache_read: Rate, + pub cache_write: Rate, + pub cache_write_1h: Rate, +} + +impl Rates { + pub const EMPTY: Self = Self { + input: Rate::Missing, + output: Rate::Missing, + cache_read: Rate::Missing, + cache_write: Rate::Missing, + cache_write_1h: Rate::Missing, + }; + + fn overlay(self, base: Self) -> Self { + Self { + input: self.input.or(base.input), + output: self.output.or(base.output), + cache_read: self.cache_read.or(base.cache_read), + cache_write: self.cache_write.or(base.cache_write), + cache_write_1h: self.cache_write_1h.or(base.cache_write_1h), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ServiceTier { + Standard, + Flex, + Priority, + Fast, + Ultrafast, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ThresholdPolicy { + Exclusive, + Inclusive, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PromptConvention { + IncludesCache, + ExcludesCache, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Usage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub cache_write_5m_tokens: Option, + pub cache_write_1h_tokens: Option, + pub prompt_convention: PromptConvention, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TierRates { + pub tier: ServiceTier, + pub rates: Rates, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ThresholdRates<'a> { + pub above_prompt_tokens: u64, + pub standard: Rates, + pub tiers: &'a [TierRates], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct OffPeakRates { + pub start_utc_minute: u16, + pub end_utc_minute: u16, + pub rates: Rates, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pricing<'a> { + pub standard: Rates, + pub tiers: &'a [TierRates], + pub thresholds: &'a [ThresholdRates<'a>], + pub off_peak: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Request { + pub usage: Usage, + pub service_tier: ServiceTier, + pub threshold_policy: ThresholdPolicy, + pub region_multiplier: Option, + pub billed_at_utc_minute: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Cost { + pub uncached_input: f64, + pub cache_read: f64, + pub cache_write_5m: f64, + pub cache_write_1h: f64, + pub output: f64, + pub multiplier: f64, + pub rates: EffectiveRates, +} + +impl Cost { + pub fn input(self) -> f64 { + (self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h) + * self.multiplier + } + + pub fn output(self) -> f64 { + self.output * self.multiplier + } + + pub fn total(self) -> f64 { + self.input() + self.output() + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EffectiveRates { + pub input: f64, + pub output: f64, + pub cache_read: f64, + pub cache_write_5m: f64, + pub cache_write_1h: f64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PricingError { + MissingInputRate, + MissingOutputRate, + InvalidRate, + InvalidRegionMultiplier, + InvalidBillingTime, + InvalidOffPeakWindow, + CacheExceedsPrompt, + InvalidCacheWriteDetails, + TokenCountOverflow, + DuplicateTier, + DuplicateThreshold, + DuplicateThresholdTier, +} + +fn selected_tier(tier: ServiceTier) -> ServiceTier { + if tier == ServiceTier::Fast { + ServiceTier::Priority + } else { + tier + } +} + +#[derive(Clone, Debug)] +struct CompiledThreshold { + above_prompt_tokens: u64, + standard: Rates, + tiers: Vec, +} + +#[derive(Clone, Debug)] +pub struct PricingPlan { + standard: Rates, + tiers: Vec, + thresholds: Vec, + off_peak: Option, +} + +fn valid_rates(rates: Rates) -> bool { + [ + rates.input, + rates.output, + rates.cache_read, + rates.cache_write, + rates.cache_write_1h, + ] + .into_iter() + .all(|rate| { + rate.value() + .is_none_or(|value| value.is_finite() && value >= 0.0) + }) +} + +fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> { + if tiers.iter().any(|entry| !valid_rates(entry.rates)) { + return Err(PricingError::InvalidRate); + } + if tiers.iter().enumerate().any(|(index, entry)| { + matches!( + entry.tier, + ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast + ) || tiers[..index] + .iter() + .any(|previous| previous.tier == entry.tier) + }) { + return Err(duplicate); + } + Ok(()) +} + +pub fn compile(pricing: &Pricing<'_>) -> Result { + if !valid_rates(pricing.standard) { + return Err(PricingError::InvalidRate); + } + validate_tiers(pricing.tiers, PricingError::DuplicateTier)?; + if let Some(window) = pricing.off_peak { + if window.start_utc_minute >= 1440 + || window.end_utc_minute > 1440 + || window.start_utc_minute >= window.end_utc_minute + { + return Err(PricingError::InvalidOffPeakWindow); + } + if !valid_rates(window.rates) { + return Err(PricingError::InvalidRate); + } + } + let mut thresholds: Vec<_> = pricing + .thresholds + .iter() + .map(|entry| { + if !valid_rates(entry.standard) { + return Err(PricingError::InvalidRate); + } + validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?; + Ok(CompiledThreshold { + above_prompt_tokens: entry.above_prompt_tokens, + standard: entry.standard, + tiers: entry.tiers.to_vec(), + }) + }) + .collect::>()?; + thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens); + if thresholds + .windows(2) + .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens) + { + return Err(PricingError::DuplicateThreshold); + } + Ok(PricingPlan { + standard: pricing.standard, + tiers: pricing.tiers.to_vec(), + thresholds, + off_peak: pricing.off_peak, + }) +} + +impl PricingPlan { + fn resolve_rates( + &self, + request: &Request, + threshold_tokens: u64, + ) -> Result { + let tier = selected_tier(request.service_tier); + let base = self + .tiers + .iter() + .find(|entry| tier != ServiceTier::Standard && entry.tier == tier) + .map_or(self.standard, |entry| entry.rates.overlay(self.standard)); + let threshold = self.thresholds.iter().rev().find(|entry| { + threshold_tokens > entry.above_prompt_tokens + || (request.threshold_policy == ThresholdPolicy::Inclusive + && threshold_tokens == entry.above_prompt_tokens) + }); + let selected = threshold.map_or(base, |entry| { + let standard = entry.standard.overlay(base); + entry + .tiers + .iter() + .find(|specific| tier != ServiceTier::Standard && specific.tier == tier) + .map_or(standard, |specific| specific.rates.overlay(standard)) + }); + match self.off_peak { + None => Ok(selected), + Some(window) => { + if window.start_utc_minute >= 1440 + || window.end_utc_minute > 1440 + || window.start_utc_minute >= window.end_utc_minute + { + return Err(PricingError::InvalidOffPeakWindow); + } + let minute = request + .billed_at_utc_minute + .ok_or(PricingError::InvalidBillingTime)?; + if minute >= 1440 { + return Err(PricingError::InvalidBillingTime); + } + if (window.start_utc_minute..window.end_utc_minute).contains(&minute) { + Ok(window.rates.overlay(selected)) + } else { + Ok(selected) + } + } + } + } + + fn checked_rate(rate: Rate, missing: PricingError) -> Result { + let value = rate.value().ok_or(missing)?; + if !value.is_finite() || value < 0.0 { + return Err(PricingError::InvalidRate); + } + Ok(value) + } + + pub fn calculate(&self, request: &Request) -> Result { + let usage = request.usage; + let cached = usage + .cache_read_tokens + .checked_add(usage.cache_write_tokens) + .ok_or(PricingError::TokenCountOverflow)?; + let (regular, threshold_tokens) = match usage.prompt_convention { + PromptConvention::IncludesCache => ( + usage + .prompt_tokens + .checked_sub(cached) + .ok_or(PricingError::CacheExceedsPrompt)?, + usage.prompt_tokens, + ), + PromptConvention::ExcludesCache => ( + usage.prompt_tokens, + usage + .prompt_tokens + .checked_add(cached) + .ok_or(PricingError::TokenCountOverflow)?, + ), + }; + let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) { + (None, None) => (usage.cache_write_tokens, 0), + (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => { + (five, one) + } + _ => return Err(PricingError::InvalidCacheWriteDetails), + }; + let rates = self.resolve_rates(request, threshold_tokens)?; + let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?; + let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?; + let read = Self::checked_rate( + rates.cache_read.or(rates.input), + PricingError::MissingInputRate, + )?; + let write = Self::checked_rate( + rates.cache_write.or(rates.input), + PricingError::MissingInputRate, + )?; + let write_1h = Self::checked_rate( + rates.cache_write_1h.or(rates.cache_write).or(rates.input), + PricingError::MissingInputRate, + )?; + let multiplier = request.region_multiplier.unwrap_or(1.0); + if !multiplier.is_finite() || multiplier <= 0.0 { + return Err(PricingError::InvalidRegionMultiplier); + } + let cost = Cost { + uncached_input: regular as f64 * input, + cache_read: usage.cache_read_tokens as f64 * read, + cache_write_5m: writes.0 as f64 * write, + cache_write_1h: writes.1 as f64 * write_1h, + output: usage.completion_tokens as f64 * output, + multiplier, + rates: EffectiveRates { + input, + output, + cache_read: read, + cache_write_5m: write, + cache_write_1h: write_1h, + }, + }; + if !cost.total().is_finite() { + return Err(PricingError::TokenCountOverflow); + } + Ok(cost) + } +} + +pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { + compile(pricing)?.calculate(request) +} diff --git a/litellm-rust/crates/cost/tests/calculation.rs b/litellm-rust/crates/cost/tests/calculation.rs new file mode 100644 index 00000000000..8cd06e74ec8 --- /dev/null +++ b/litellm-rust/crates/cost/tests/calculation.rs @@ -0,0 +1,458 @@ +use litellm_cost::{ + OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier, + ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile, +}; + +fn rates(input: Rate, output: Rate) -> Rates { + Rates { + input, + output, + ..Rates::EMPTY + } +} + +fn request() -> Request { + Request { + usage: Usage { + prompt_tokens: 100, + completion_tokens: 20, + cache_read_tokens: 25, + cache_write_tokens: 10, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + } +} + +fn pricing(standard: Rates) -> Pricing<'static> { + Pricing { + standard, + tiers: &[], + thresholds: &[], + off_peak: None, + } +} + +#[test] +fn breakdown_and_total_agree() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let result = calculate(&pricing(standard), &request()).unwrap(); + assert_eq!(result.uncached_input, 65.0 * 2.0); + assert_eq!(result.cache_read, 25.0 * 0.5); + assert_eq!(result.cache_write_5m, 10.0 * 3.0); + assert_eq!(result.output(), 20.0 * 4.0); + assert_eq!(result.total(), result.input() + result.output()); + assert_eq!(result.rates.cache_read, 0.5); +} + +#[test] +fn absent_null_and_zero_cache_rates_are_distinct() { + let base = rates(Rate::Value(2.0), Rate::Value(4.0)); + for read in [Rate::Missing, Rate::Null] { + let standard = Rates { + cache_read: read, + ..base + }; + assert_eq!( + calculate(&pricing(standard), &request()).unwrap().input(), + 200.0 + ); + } + let standard = Rates { + cache_read: Rate::Value(0.0), + cache_write: Rate::Value(0.0), + ..base + }; + assert_eq!( + calculate(&pricing(standard), &request()).unwrap().input(), + 130.0 + ); +} + +#[test] +fn equivalent_prompt_conventions_select_the_same_threshold() { + let threshold = ThresholdRates { + above_prompt_tokens: 90, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[threshold], + off_peak: None, + }; + let included = request(); + let excluded = Request { + usage: Usage { + prompt_tokens: 65, + prompt_convention: PromptConvention::ExcludesCache, + ..included.usage + }, + ..included + }; + let plan = compile(&specification).unwrap(); + assert_eq!(plan.calculate(&included), plan.calculate(&excluded)); + assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0); +} + +#[test] +fn split_writes_and_invalid_accounting() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + cache_write_1h: Rate::Value(5.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let base = request(); + let split = Request { + usage: Usage { + cache_write_5m_tokens: Some(4), + cache_write_1h_tokens: Some(6), + ..base.usage + }, + ..base + }; + let result = calculate(&pricing(standard), &split).unwrap(); + assert_eq!(result.cache_write_5m, 12.0); + assert_eq!(result.cache_write_1h, 30.0); + let overlapping = Request { + usage: Usage { + prompt_tokens: 30, + ..split.usage + }, + ..split + }; + assert_eq!( + calculate(&pricing(standard), &overlapping), + Err(PricingError::CacheExceedsPrompt) + ); + let incomplete = Request { + usage: Usage { + cache_write_1h_tokens: None, + ..split.usage + }, + ..split + }; + assert_eq!( + calculate(&pricing(standard), &incomplete), + Err(PricingError::InvalidCacheWriteDetails) + ); +} + +#[test] +fn threshold_tiers_and_boundaries() { + let priority = TierRates { + tier: ServiceTier::Priority, + rates: rates(Rate::Value(3.0), Rate::Missing), + }; + let threshold = ThresholdRates { + above_prompt_tokens: 100, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[ + TierRates { + tier: ServiceTier::Priority, + rates: rates(Rate::Value(7.0), Rate::Missing), + }, + TierRates { + tier: ServiceTier::Flex, + rates: rates(Rate::Value(6.0), Rate::Missing), + }, + ], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[priority], + thresholds: &[threshold], + off_peak: None, + }; + let base = request(); + let no_cache = Request { + usage: Usage { + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + let fast = Request { + service_tier: ServiceTier::Fast, + ..no_cache + }; + let inclusive = Request { + threshold_policy: ThresholdPolicy::Inclusive, + ..fast + }; + let flex = Request { + service_tier: ServiceTier::Flex, + ..inclusive + }; + assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0); + assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0); + assert_eq!( + calculate(&specification, &inclusive).unwrap().input(), + 700.0 + ); + assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0); +} + +#[test] +fn compile_rejects_ambiguous_rates() { + let duplicate = ThresholdRates { + above_prompt_tokens: 100, + standard: Rates::EMPTY, + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(1.0), Rate::Value(1.0)), + tiers: &[], + thresholds: &[duplicate, duplicate], + off_peak: None, + }; + assert_eq!( + compile(&specification).err(), + Some(PricingError::DuplicateThreshold) + ); + let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0))); + assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate)); +} + +#[test] +fn off_peak_is_one_non_wrapping_utc_window() { + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[], + off_peak: Some(OffPeakRates { + start_utc_minute: 60, + end_utc_minute: 120, + rates: rates(Rate::Value(1.0), Rate::Value(2.0)), + }), + }; + let base = request(); + let start = Request { + billed_at_utc_minute: Some(60), + ..base + }; + let end = Request { + billed_at_utc_minute: Some(120), + ..base + }; + assert_eq!( + calculate(&specification, &base), + Err(PricingError::InvalidBillingTime) + ); + assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0); + assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0); +} + +#[test] +fn missing_rates_and_free_rates_remain_distinct() { + let base = request(); + let empty = Request { + usage: Usage { + prompt_tokens: 0, + completion_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + assert_eq!( + calculate(&pricing(Rates::EMPTY), &empty), + Err(PricingError::MissingInputRate) + ); + assert_eq!( + calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty), + Err(PricingError::MissingOutputRate) + ); + assert_eq!( + calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty) + .unwrap() + .total(), + 0.0 + ); +} + +#[test] +fn matches_executed_python_reference_cases() { + for row in include_str!("python_reference.tsv") + .lines() + .filter(|line| !line.starts_with('#')) + { + let fields: Vec<_> = row.split('\t').collect(); + let count = |index: usize| fields[index].parse::().unwrap(); + let number = |index: usize| fields[index].parse::().unwrap(); + let optional_rate = |index: usize| { + if fields[index].is_empty() { + Rate::Missing + } else { + Rate::Value(number(index)) + } + }; + let threshold = ThresholdRates { + above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) }, + standard: rates(optional_rate(10), optional_rate(11)), + tiers: &[], + }; + let thresholds = if fields[9].is_empty() { + &[][..] + } else { + std::slice::from_ref(&threshold) + }; + let specification = Pricing { + standard: Rates { + cache_read: optional_rate(7), + cache_write: optional_rate(8), + ..rates(Rate::Value(number(5)), Rate::Value(number(6))) + }, + tiers: &[], + thresholds, + off_peak: None, + }; + let base = request(); + let input = Request { + usage: Usage { + prompt_tokens: count(1), + completion_tokens: count(2), + cache_read_tokens: count(3), + cache_write_tokens: count(4), + ..base.usage + }, + ..base + }; + let actual = calculate(&specification, &input).unwrap(); + assert_eq!(actual.input(), number(12), "{}", fields[0]); + assert_eq!(actual.output(), number(13), "{}", fields[0]); + } +} + +proptest::proptest! { + #[test] + fn equivalent_usage_conventions_and_breakdown_agree( + regular in 0_u64..1000, + read in 0_u64..1000, + write in 0_u64..1000, + output in 0_u64..1000, + ) { + let threshold = ThresholdRates { + above_prompt_tokens: 1000, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[threshold], + off_peak: None, + }; + let base = request(); + let included = Request { + usage: Usage { + prompt_tokens: regular + read + write, + completion_tokens: output, + cache_read_tokens: read, + cache_write_tokens: write, + ..base.usage + }, + ..base + }; + let excluded = Request { + usage: Usage { + prompt_tokens: regular, + prompt_convention: PromptConvention::ExcludesCache, + ..included.usage + }, + ..included + }; + let plan = compile(&specification).unwrap(); + let left = plan.calculate(&included).unwrap(); + let right = plan.calculate(&excluded).unwrap(); + proptest::prop_assert_eq!(left, right); + proptest::prop_assert_eq!(left.total(), left.input() + left.output()); + } +} + +#[test] +fn regional_multiplier_applies_after_input_components_are_summed() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let base = request(); + let regional = Request { + region_multiplier: Some(1.1), + ..base + }; + let result = calculate(&pricing(standard), ®ional).unwrap(); + assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1); + assert_eq!(result.output(), 20.0 * 4.0 * 1.1); + let invalid = Request { + region_multiplier: Some(f64::NAN), + ..base + }; + assert_eq!( + calculate(&pricing(standard), &invalid), + Err(PricingError::InvalidRegionMultiplier) + ); +} + +#[test] +fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() { + let high = ThresholdRates { + above_prompt_tokens: 200, + standard: rates(Rate::Value(7.0), Rate::Missing), + tiers: &[], + }; + let low = ThresholdRates { + above_prompt_tokens: 100, + standard: rates(Rate::Value(5.0), Rate::Missing), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[high, low], + off_peak: None, + }; + let base = request(); + let above_both = Request { + usage: Usage { + prompt_tokens: 201, + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + assert_eq!( + compile(&specification) + .unwrap() + .calculate(&above_both) + .unwrap() + .rates + .input, + 7.0 + ); + let duplicate = TierRates { + tier: ServiceTier::Flex, + rates: Rates::EMPTY, + }; + let invalid = Pricing { + tiers: &[duplicate, duplicate], + thresholds: &[], + ..specification + }; + assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier)); +} diff --git a/litellm-rust/crates/cost/tests/generate_python_reference.py b/litellm-rust/crates/cost/tests/generate_python_reference.py new file mode 100644 index 00000000000..4a179555650 --- /dev/null +++ b/litellm-rust/crates/cost/tests/generate_python_reference.py @@ -0,0 +1,96 @@ +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import Usage + + +@dataclass(frozen=True, slots=True) +class Case: + name: str + prompt: int + completion: int + cache_read: int + cache_write: int + input_rate: float + output_rate: float + cache_read_rate: float | None = None + cache_write_rate: float | None = None + threshold: int | None = None + threshold_input_rate: float | None = None + threshold_output_rate: float | None = None + + +CASES = ( + Case("ordinary", 100, 20, 0, 0, 2.0, 4.0), + Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0), + Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0), + Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0), + Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0), + Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0), + Case( + "threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0 + ), + Case( + "cache_threshold_above", + 101, + 20, + 25, + 10, + 2.0, + 4.0, + threshold=100, + threshold_input_rate=5.0, + threshold_output_rate=8.0, + ), +) + + +def reference(case: Case) -> tuple[float, float]: + info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate} + if case.cache_read_rate is not None: + info["cache_read_input_token_cost"] = case.cache_read_rate + if case.cache_write_rate is not None: + info["cache_creation_input_token_cost"] = case.cache_write_rate + if case.threshold is not None: + info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate + info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate + details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write} + usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details) + return generic_cost_per_token( + model="synthetic", + usage=usage, + custom_llm_provider="openai", + model_info=info, + ) + + +def main() -> None: + revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip() + rows = ("# Python reference commit: " + revision,) + tuple( + "\t".join( + str(value) if value is not None else "" + for value in ( + case.name, + case.prompt, + case.completion, + case.cache_read, + case.cache_write, + case.input_rate, + case.output_rate, + case.cache_read_rate, + case.cache_write_rate, + case.threshold, + case.threshold_input_rate, + case.threshold_output_rate, + *reference(case), + ) + ) + for case in CASES + ) + Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n") + + +if __name__ == "__main__": + main() diff --git a/litellm-rust/crates/cost/tests/python_reference.tsv b/litellm-rust/crates/cost/tests/python_reference.tsv new file mode 100644 index 00000000000..5abb1329a61 --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_reference.tsv @@ -0,0 +1,9 @@ +# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627 +ordinary 100 20 0 0 2.0 4.0 200.0 80.0 +cache_fallback 100 20 25 10 2.0 4.0 200.0 80.0 +cache_specific 100 20 25 10 2.0 4.0 0.5 3.0 172.5 80.0 +free_cache 100 20 25 10 2.0 4.0 0.0 0.0 130.0 80.0 +threshold_below 99 20 0 0 2.0 4.0 100 5.0 8.0 198.0 80.0 +threshold_at 100 20 0 0 2.0 4.0 100 5.0 8.0 200.0 80.0 +threshold_above 101 20 0 0 2.0 4.0 100 5.0 8.0 505.0 160.0 +cache_threshold_above 101 20 25 10 2.0 4.0 100 5.0 8.0 505.0 160.0 From 392e80717200b36b083ce859c866af75d7721569 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:56:50 -0700 Subject: [PATCH 14/17] feat(logging): add normalized_error cluster key to error_information (#41715) * feat(logging): add normalized_error cluster key to error_information Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): stop classifying parameter length errors as context window errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): assert failure spend rows share normalized_error across provider wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): map agent model access denials and ignore non-string proxy error types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(logging): cover budget exceeded errors with custom wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster router no-healthy and provider-budget wording correctly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): let the exception class win over router fallback wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster peer closed connection errors as provider connection errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster tag routing denials as 403_MODEL_ACCESS_DENIED Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- .../litellm_core_utils/error_normalization.py | 195 +++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/types/utils.py | 1 + .../coverage_registry/quota_management.yaml | 1 + tests/e2e/models.py | 8 + .../spend_tracking/test_spend_tracking_e2e.py | 41 ++++ .../test_error_normalization.py | 231 ++++++++++++++++++ 7 files changed, 479 insertions(+) create mode 100644 litellm/litellm_core_utils/error_normalization.py create mode 100644 tests/test_litellm/litellm_core_utils/test_error_normalization.py diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py new file mode 100644 index 00000000000..2a9ff748899 --- /dev/null +++ b/litellm/litellm_core_utils/error_normalization.py @@ -0,0 +1,195 @@ +""" +Map any exception litellm logs to one stable ``normalized_error`` code so dashboards can cluster +failures without parsing free-text messages that embed team names, token counts, model names, etc. +""" + +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +from litellm.exceptions import ( + APIConnectionError, + AuthenticationError, + BadGatewayError, + BadRequestError, + BlockedPiiEntityError, + BudgetExceededError, + ContentPolicyViolationError, + ContextWindowExceededError, + GuardrailRaisedException, + InternalServerError, + MidStreamFallbackError, + NotFoundError, + PermissionDeniedError, + RateLimitError, + RateLimitType, + ServiceUnavailableError, + Timeout, + UnprocessableEntityError, + UnsupportedParamsError, +) + +RATE_LIMIT_EXCEEDED: Final = "429_RATE_LIMIT_EXCEEDED" +BUDGET_EXCEEDED: Final = "429_BUDGET_EXCEEDED" +NO_HEALTHY_DEPLOYMENTS: Final = "429_NO_HEALTHY_DEPLOYMENTS" +AUTHENTICATION_FAILED: Final = "401_AUTHENTICATION_FAILED" +MODEL_ACCESS_DENIED: Final = "403_MODEL_ACCESS_DENIED" +PERMISSION_DENIED: Final = "403_PERMISSION_DENIED" +MISSING_REQUIRED_PARAMETER: Final = "400_MISSING_REQUIRED_PARAMETER" +INVALID_PARAMETER_VALUE: Final = "400_INVALID_PARAMETER_VALUE" +CONTEXT_WINDOW_EXCEEDED: Final = "400_CONTEXT_WINDOW_EXCEEDED" +CONTENT_POLICY_VIOLATION: Final = "400_CONTENT_POLICY_VIOLATION" +INVALID_REQUEST: Final = "400_INVALID_REQUEST" +RESOURCE_NOT_FOUND: Final = "404_RESOURCE_NOT_FOUND" +UPSTREAM_TIMEOUT: Final = "408_UPSTREAM_TIMEOUT" +PROVIDER_CONNECTION_ERROR: Final = "500_PROVIDER_CONNECTION_ERROR" +PROVIDER_OVERLOADED: Final = "503_PROVIDER_OVERLOADED" +PROVIDER_INTERNAL_ERROR: Final = "500_PROVIDER_INTERNAL_ERROR" +ROUTER_NO_FALLBACK: Final = "500_ROUTER_NO_FALLBACK" +ROUTER_FALLBACK_FAILURE: Final = "500_ROUTER_FALLBACK_FAILURE" +UPSTREAM_PASSTHROUGH: Final = "500_UPSTREAM_PASSTHROUGH" +UNSUPPORTED_OPERATION: Final = "500_UNSUPPORTED_OPERATION" +INTERNAL_STATE_ERROR: Final = "500_INTERNAL_STATE_ERROR" +UNCLASSIFIED: Final = "UNCLASSIFIED" + + +@runtime_checkable +class _HasProxyErrorType(Protocol): + type: str + + +_MESSAGE_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + ( + re.compile(r"budget has been exceeded|max budget|exceeded.*budget|crossed budget", re.IGNORECASE), + BUDGET_EXCEEDED, + ), + (re.compile(r"no healthy deployments?|no deployments available", re.IGNORECASE), NO_HEALTHY_DEPLOYMENTS), + (re.compile(r"not allowed to access model due to tags configuration", re.IGNORECASE), MODEL_ACCESS_DENIED), + (re.compile(r"upstream passthrough request failed", re.IGNORECASE), UPSTREAM_PASSTHROUGH), + (re.compile(r"is not supported for provider|not implemented", re.IGNORECASE), UNSUPPORTED_OPERATION), + ( + re.compile(r"context window|context length|(prompt|input) is too long|tokens? ?> ?\d+ ?maximum", re.IGNORECASE), + CONTEXT_WINDOW_EXCEEDED, + ), + (re.compile(r"missing required parameter|field required", re.IGNORECASE), MISSING_REQUIRED_PARAMETER), + (re.compile(r"overloaded|unable to process your request", re.IGNORECASE), PROVIDER_OVERLOADED), + ( + re.compile( + r"connection error|APIConnectionError|TransferEncodingError|payload is not completed|connection reset" + r"|peer closed connection|incomplete chunked read", + re.IGNORECASE, + ), + PROVIDER_CONNECTION_ERROR, + ), + (re.compile(r"timed? ?out", re.IGNORECASE), UPSTREAM_TIMEOUT), +) + +_ROUTER_WRAPPER_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + (re.compile(r"no fallback model group found", re.IGNORECASE), ROUTER_NO_FALLBACK), + (re.compile(r"error doing the fallback|MidStreamFallbackError", re.IGNORECASE), ROUTER_FALLBACK_FAILURE), +) + +_PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "budget_exceeded": BUDGET_EXCEEDED, + "auth_error": AUTHENTICATION_FAILED, + "expired_key": AUTHENTICATION_FAILED, + "token_not_found_in_db": AUTHENTICATION_FAILED, + "auth_provider_unavailable": AUTHENTICATION_FAILED, + "key_model_access_denied": MODEL_ACCESS_DENIED, + "team_model_access_denied": MODEL_ACCESS_DENIED, + "user_model_access_denied": MODEL_ACCESS_DENIED, + "org_model_access_denied": MODEL_ACCESS_DENIED, + "project_model_access_denied": MODEL_ACCESS_DENIED, + "agent_model_access_denied": MODEL_ACCESS_DENIED, + "key_vector_store_access_denied": PERMISSION_DENIED, + "team_vector_store_access_denied": PERMISSION_DENIED, + "org_vector_store_access_denied": PERMISSION_DENIED, + "tool_access_denied": PERMISSION_DENIED, + "team_member_permission_error": PERMISSION_DENIED, + "not_found_error": RESOURCE_NOT_FOUND, + } +) + +_STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "400": INVALID_REQUEST, + "401": AUTHENTICATION_FAILED, + "403": PERMISSION_DENIED, + "404": RESOURCE_NOT_FOUND, + "408": UPSTREAM_TIMEOUT, + "422": INVALID_PARAMETER_VALUE, + "429": RATE_LIMIT_EXCEEDED, + "500": PROVIDER_INTERNAL_ERROR, + "502": PROVIDER_INTERNAL_ERROR, + "503": PROVIDER_OVERLOADED, + "504": UPSTREAM_TIMEOUT, + } +) + +_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( + TypeError, + KeyError, + AttributeError, + IndexError, + RuntimeError, + AssertionError, + ZeroDivisionError, +) + +_CLASS_CODE_TABLE: Final[tuple[tuple[tuple[type[BaseException], ...], str], ...]] = ( + ((AuthenticationError,), AUTHENTICATION_FAILED), + ((PermissionDeniedError,), PERMISSION_DENIED), + ((ContextWindowExceededError,), CONTEXT_WINDOW_EXCEEDED), + ((ContentPolicyViolationError, GuardrailRaisedException, BlockedPiiEntityError), CONTENT_POLICY_VIOLATION), + ((UnsupportedParamsError,), INVALID_PARAMETER_VALUE), + ((NotFoundError,), RESOURCE_NOT_FOUND), + ((Timeout,), UPSTREAM_TIMEOUT), + ((MidStreamFallbackError,), ROUTER_FALLBACK_FAILURE), + ((APIConnectionError,), PROVIDER_CONNECTION_ERROR), + ((ServiceUnavailableError,), PROVIDER_OVERLOADED), + ((InternalServerError, BadGatewayError), PROVIDER_INTERNAL_ERROR), + ((BadRequestError, UnprocessableEntityError), INVALID_REQUEST), + ((NotImplementedError,), UNSUPPORTED_OPERATION), +) + + +def _classify_by_message(message: str, patterns: tuple[tuple[re.Pattern[str], str], ...]) -> str | None: + return next((code for pattern, code in patterns if pattern.search(message)), None) + + +def _classify_by_class(exc: Exception) -> str | None: + if isinstance(exc, BudgetExceededError): + return BUDGET_EXCEEDED + if isinstance(exc, RateLimitError): + return BUDGET_EXCEEDED if exc.rate_limit_type == RateLimitType.BUDGET.value else RATE_LIMIT_EXCEEDED + for exc_types, code in _CLASS_CODE_TABLE: + if isinstance(exc, exc_types): + return code + if isinstance(exc, _INTERNAL_STATE_EXCEPTIONS): + return INTERNAL_STATE_ERROR + return None + + +def normalize_error(exc: Exception | None, status_code: str, message: str) -> str | None: + """ + Return a stable cluster key for ``exc``. ``status_code`` and ``message`` are the values + ``get_error_information`` already extracted, so the same exception always yields the same code. + """ + if exc is None: + return None + proxy_type: Final = exc.type if isinstance(exc, _HasProxyErrorType) else None + by_proxy_type: Final = _PROXY_ERROR_TYPE_MAP.get(proxy_type) if isinstance(proxy_type, str) else None + if by_proxy_type is not None: + return by_proxy_type + by_message: Final = _classify_by_message(message, _MESSAGE_PATTERNS) + if by_message is not None: + return by_message + by_class: Final = _classify_by_class(exc) + if by_class is not None: + return by_class + by_router_wrapper: Final = _classify_by_message(message, _ROUTER_WRAPPER_PATTERNS) + if by_router_wrapper is not None: + return by_router_wrapper + return _STATUS_CODE_MAP.get(status_code, UNCLASSIFIED) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6e5a37a7226..8e28a0d543d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.litellm_core_utils.core_helpers import ( reconstruct_model_name, set_response_cost_in_hidden_params, ) +from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -6100,6 +6101,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + normalized_error=normalize_error(original_exception, error_status, error_message), ) @staticmethod diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f2c8f0e7044..dfc98a9d89d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3256,6 +3256,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + normalized_error: ReadOnly[str | None] class GuardrailMode(TypedDict, total=False): diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 5ea48fdcd9d..5740a878608 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -51,6 +51,7 @@ - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} +- {id: quota_management.spend_tracking.failure.writes_normalized_error, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_normalized_error], exercised_on: [chat_completions], source: "litellm_core_utils/error_normalization.py", rationale: "Failure rows carry a stable metadata.error_information.normalized_error key next to the unchanged error_message, so two upstream auth failures with different provider wording share one cluster key a dashboard can group by"} - {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index bc81d1015ba..bd7f5171172 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -937,10 +937,18 @@ class GuardrailRunRecord(BaseModel): guardrail_response: object | None = None +class SpendLogErrorInformation(BaseModel): + error_code: str | None = None + error_class: str | None = None + error_message: str | None = None + normalized_error: str | None = None + + class SpendLogMetadata(BaseModel): user_api_key_alias: str | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None + error_information: SpendLogErrorInformation | None = None class SpendLogRow(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 6ca6f8cee55..286421e2e3f 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -499,6 +499,47 @@ def test_failure_call_writes_failure_status_row( assert (failure_row.spend or 0) == 0.0, "failed call must not be charged" +@pytest.mark.covers("quota_management.spend_tracking.failure.writes_normalized_error") +def test_failure_rows_share_normalized_error_across_provider_wording( + client: SpendClient, resources: ResourceManager, scoped_key: str +) -> None: + """Two upstream auth failures with different provider wording land as failure rows + whose metadata.error_information keeps each provider's own error_message and + carries the same stable normalized_error cluster key.""" + marker = unique_marker() + deployments: Final = ( + (f"e2e-norm-openai-{marker}", "openai/gpt-5.5"), + (f"e2e-norm-anthropic-{marker}", "anthropic/claude-haiku-4-5"), + ) + for name, provider_model in deployments: + model_id = client.proxy.create_model( + name, LiteLLMParamsBody(model=provider_model, api_key=f"sk-invalid-{marker}") + ) + resources.defer(lambda model_id=model_id: client.proxy.delete_model(model_id)) + result = client.chat(scoped_key, name, f"normalize failure {marker}", max_tokens=1) + assert not is_ok(result), f"{name}: invalid upstream key must fail the call, got {result}" + + rows = client.poll_logs_for_key( + scoped_key, + min_rows=2, + predicate=lambda rs: sum(1 for r in rs if r.status == "failure") >= 2, + ) + failure_rows = [r for r in rows if r.status == "failure"] + assert len(failure_rows) == 2, f"expected one failure row per deployment: {_summarize(rows)}" + + infos = [r.metadata.error_information if r.metadata else None for r in failure_rows] + assert all(info is not None for info in infos), ( + f"failure rows must carry metadata.error_information: {[r.model_dump() for r in failure_rows]}" + ) + messages = {info.error_message for info in infos if info is not None} + assert len(messages) == 2, f"provider wording must stay distinct in error_message: {messages}" + normalized = {info.normalized_error for info in infos if info is not None} + assert normalized == {"401_AUTHENTICATION_FAILED"}, ( + f"both auth failures must share one normalized_error cluster key; saw {normalized} " + f"for messages {messages}" + ) + + @pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") def test_pre_call_rejection_row_attributes_provider_and_model_id( client: SpendClient, resources: ResourceManager diff --git a/tests/test_litellm/litellm_core_utils/test_error_normalization.py b/tests/test_litellm/litellm_core_utils/test_error_normalization.py new file mode 100644 index 00000000000..9d5469ddb4c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_error_normalization.py @@ -0,0 +1,231 @@ +import httpx +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.error_normalization import normalize_error +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.types.router import RouterErrors + +_RESPONSE = httpx.Response(status_code=500, request=httpx.Request("POST", "https://example.invalid")) + + +def _proxy_exc(message: str, error_type: str, code: int) -> ProxyException: + return ProxyException(message=message, type=error_type, param=None, code=code) + + +@pytest.mark.parametrize( + ("messages", "expected"), + [ + ( + ( + _proxy_exc("Rate limit exceeded for team X. Reset at 10:01", "rate_limit_error", 429), + _proxy_exc("Rate limit exceeded for team Y. Reset at 10:02", "rate_limit_error", 429), + ), + "429_RATE_LIMIT_EXCEEDED", + ), + ( + ( + litellm.BudgetExceededError(current_cost=3501.85, max_budget=3500), + _proxy_exc( + "User=abc, Current cost=1000.03, Max budget=1000", ProxyErrorTypes.budget_exceeded.value, 400 + ), + litellm.RateLimitError( + "budget", + llm_provider="openai", + model="gpt", + rate_limit_type=litellm.exceptions.RateLimitType.BUDGET, + ), + ), + "429_BUDGET_EXCEEDED", + ), + ( + ( + _proxy_exc("Token Expired", ProxyErrorTypes.expired_key.value, 401), + _proxy_exc("Malformed API Key", ProxyErrorTypes.auth_error.value, 401), + litellm.AuthenticationError("Signature verification failed", llm_provider="azure", model="gpt"), + ), + "401_AUTHENTICATION_FAILED", + ), + ( + ( + _proxy_exc("No team has access to gpt-5.5-mini", ProxyErrorTypes.team_model_access_denied.value, 401), + _proxy_exc("key not allowed to access claude", ProxyErrorTypes.key_model_access_denied.value, 401), + ValueError( + "Not allowed to access model due to tags configuration. Passed model=gpt-5.5 and tags=['team-a']" + ), + ), + "403_MODEL_ACCESS_DENIED", + ), + ( + ( + _proxy_exc("Missing required parameter: messages", ProxyErrorTypes.bad_request_error.value, 400), + litellm.BadRequestError("Missing required parameter: input", llm_provider="openai", model="gpt"), + ), + "400_MISSING_REQUIRED_PARAMETER", + ), + ( + ( + litellm.ContextWindowExceededError( + "1002823 tokens > 1000000 maximum", model="g", llm_provider="vertex" + ), + litellm.BadRequestError("Input is too long for requested model", llm_provider="anthropic", model="c"), + ), + "400_CONTEXT_WINDOW_EXCEEDED", + ), + ( + ( + litellm.NotFoundError("Response id xxx not found", llm_provider="openai", model="gpt"), + _proxy_exc("No vector store found with id abc", ProxyErrorTypes.not_found_error.value, 404), + ), + "404_RESOURCE_NOT_FOUND", + ), + ( + ( + litellm.APIConnectionError("Connection error", llm_provider="openai", model="gpt"), + litellm.InternalServerError("TransferEncodingError", llm_provider="openai", model="gpt"), + litellm.APIError(500, "Response payload is not completed", llm_provider="openai", model="gpt"), + httpx.RemoteProtocolError( + "peer closed connection without sending complete message body (incomplete chunked read)" + ), + ), + "500_PROVIDER_CONNECTION_ERROR", + ), + ( + ( + litellm.ServiceUnavailableError("server_is_overloaded", llm_provider="anthropic", model="c"), + litellm.InternalServerError( + "Bedrock is unable to process your request", llm_provider="bedrock", model="c" + ), + litellm.APIError(529, "Overloaded", llm_provider="anthropic", model="c"), + ), + "503_PROVIDER_OVERLOADED", + ), + ( + ( + litellm.InternalServerError( + "The server had an error while processing your request", llm_provider="openai", model="gpt" + ), + litellm.APIError(500, "server_error", llm_provider="openai", model="gpt"), + ), + "500_PROVIDER_INTERNAL_ERROR", + ), + ( + ( + _proxy_exc("No fallback model group found for gpt-5.6", "internal_server_error", 500), + _proxy_exc("No fallback model group found for claude-46-sonnet", "internal_server_error", 500), + ), + "500_ROUTER_NO_FALLBACK", + ), + ( + ( + _proxy_exc("Error doing the fallback: RateLimitError", "internal_server_error", 500), + MidStreamFallbackError( + "stream died", model="gpt", llm_provider="openai", original_exception=ValueError("boom") + ), + ), + "500_ROUTER_FALLBACK_FAILURE", + ), + ( + ( + TypeError("cannot pickle '_thread.RLock' object"), + RuntimeError("dictionary changed size during iteration"), + TypeError("'NoneType' object is not iterable"), + ), + "500_INTERNAL_STATE_ERROR", + ), + ( + ( + litellm.Timeout("Timeout on reading data from socket", model="gpt", llm_provider="openai"), + litellm.APIError(504, "Request timed out", llm_provider="openai", model="gpt"), + ), + "408_UPSTREAM_TIMEOUT", + ), + ( + ( + _proxy_exc("500: Upstream passthrough request failed", "internal_server_error", 500), + _proxy_exc("503: Upstream passthrough request failed", "internal_server_error", 503), + ), + "500_UPSTREAM_PASSTHROUGH", + ), + ( + ( + _proxy_exc("OCR is not supported for provider openai", "internal_server_error", 500), + NotImplementedError("rerank"), + ), + "500_UNSUPPORTED_OPERATION", + ), + ], +) +def test_variants_of_one_failure_share_a_normalized_error(messages: tuple[Exception, ...], expected: str) -> None: + normalized = {StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] for exc in messages} + assert normalized == {expected} + + +def test_router_no_healthy_deployment_wording_clusters_as_no_healthy_deployments() -> None: + for message in (RouterErrors.no_healthy_deployments.value, "No healthy deployments found."): + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_NO_HEALTHY_DEPLOYMENTS", message + + +def test_provider_budget_routing_wording_clusters_as_budget_exceeded() -> None: + message = RouterErrors.no_deployments_with_provider_budget_routing.value + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_BUDGET_EXCEEDED" + + +def test_router_fallback_wording_does_not_hide_the_wrapped_exception_class() -> None: + provider_message = "litellm.AuthenticationError: OpenAIException - Incorrect API key provided" + exc = litellm.AuthenticationError( + provider_message + "\nNo fallback model group found for lookup_groups=['x']", + llm_provider="openai", + model="gpt", + ) + assert normalize_error(exc, "401", str(exc)) == "401_AUTHENTICATION_FAILED" + wrapped = litellm.AuthenticationError( + "Error doing the fallback: " + provider_message, llm_provider="openai", model="gpt" + ) + assert normalize_error(wrapped, "401", str(wrapped)) == "401_AUTHENTICATION_FAILED" + + +def test_parameter_length_error_is_not_a_context_window_error() -> None: + exc = litellm.BadRequestError("string too long: 'user' max 64 chars", llm_provider="openai", model="gpt") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "400_INVALID_REQUEST" + + +def test_no_exception_has_no_normalized_error() -> None: + assert StandardLoggingPayloadSetup.get_error_information(None)["normalized_error"] is None + + +def test_unknown_exception_falls_back_to_status_then_unclassified() -> None: + assert normalize_error(Exception("x"), "429", "x") == "429_RATE_LIMIT_EXCEEDED" + assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED" + + +def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None: + exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED" + + +def test_every_model_access_denied_proxy_type_shares_one_cluster() -> None: + access_denied_types = tuple(t for t in ProxyErrorTypes if t.value.endswith("_model_access_denied")) + assert len(access_denied_types) >= 6, access_denied_types + codes = {normalize_error(_proxy_exc("denied", t.value, 403), "403", "denied") for t in access_denied_types} + assert codes == {"403_MODEL_ACCESS_DENIED"}, codes + + +def test_non_string_type_attribute_falls_through_to_status() -> None: + class _OddType(Exception): + type = {"kind": "odd"} + + assert normalize_error(_OddType("odd"), "500", "odd") == "500_PROVIDER_INTERNAL_ERROR" + + +def test_normalized_error_never_embeds_dynamic_parts() -> None: + exc = _proxy_exc( + "No team has access to anthropic.claude-sonnet-4-5", ProxyErrorTypes.team_model_access_denied.value, 401 + ) + info = StandardLoggingPayloadSetup.get_error_information(exc) + assert info["error_message"] == "No team has access to anthropic.claude-sonnet-4-5" + assert "claude" not in (info["normalized_error"] or "") From 3c3803f37afa488a5fad08b2fc2f39d1c36f740c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:04:44 -0700 Subject: [PATCH 15/17] test(bedrock): point unit tests at model ids still in the cost map (#42606) #42521 retired the cohere.command-r ids from the cost map, and a bare Bedrock id resolves its provider through that map, so test_model_group_info and the cohere cases in test_bedrock_dynamic_auth_params_unit_tests failed with LLM Provider NOT provided. The completion tests keep the cohere invoke request and URL assertions through the bedrock/ prefix, the bare parametrize entry moves to amazon.nova-2-lite-v1:0 with the mock response shape picked by the real Bedrock route, and the router test builds its group info from bedrock/amazon.nova-2-lite-v1:0. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../test_bedrock_dynamic_auth_params_unit_tests.py | 11 ++++++----- tests/local_testing/test_router.py | 9 ++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py index 2d1d2815026..184a0ae0749 100644 --- a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py +++ b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py @@ -9,6 +9,7 @@ import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler from unittest.mock import Mock from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockModelInfo @@ -42,7 +43,7 @@ def test_bedrock_completion_with_region_name(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_region_name="us-west-12", client=client, @@ -98,7 +99,7 @@ def test_bedrock_completion_with_dynamic_authentication_params(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_access_key_id="dynamically_generated_access_key_id", aws_secret_access_key="dynamically_generated_secret_access_key", @@ -146,7 +147,7 @@ def test_bedrock_completion_with_dynamic_bedrock_runtime_endpoint(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com", client=client, @@ -179,7 +180,7 @@ class DummyCredentials: "model", [ "bedrock/converse/cohere.command-r-v1:0", - "cohere.command-r-v1:0", + "amazon.nova-2-lite-v1:0", "bedrock/cohere.command-r-v1:0", "bedrock/invoke/cohere.command-r-v1:0", ], @@ -250,7 +251,7 @@ def test_dynamic_aws_params_propagation(model, param_name, param_value, expected "finish_reason": "COMPLETE", } ) - if "converse" in model: + if BedrockModelInfo.get_bedrock_route(model) == "converse": mock_response.text = json.dumps( { "output": { diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index bd0a9bf8df4..4c62c28530d 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1284,15 +1284,18 @@ def test_model_group_info(): router = Router( model_list=[ { - "model_name": "command-r-plus", - "litellm_params": {"model": "cohere.command-r-plus-v1:0"}, + "model_name": "nova-2-lite", + "litellm_params": {"model": "bedrock/amazon.nova-2-lite-v1:0"}, } ] ) - response = router.get_model_group_info(model_group="command-r-plus") + response = router.get_model_group_info(model_group="nova-2-lite") assert response is not None + assert response.model_group == "nova-2-lite" + assert response.providers == ["bedrock"] + assert response.max_input_tokens is not None def test_consistent_model_id(): From dd327156c80865c58f0eff5599c987020ed545f7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:07:02 +0000 Subject: [PATCH 16/17] feat(rust): add immutable model catalog crate (#42605) * feat(rust): add immutable model catalog crate * feat(rust): add typed model info mirror, error module, and schema feature Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rust): split catalog module, rstest tests, and repo data parity tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 38 + litellm-rust/crates/model-catalog/Cargo.toml | 25 + litellm-rust/crates/model-catalog/README.md | 25 + .../crates/model-catalog/benches/catalog.rs | 21 + .../crates/model-catalog/src/catalog.rs | 241 +++++++ .../crates/model-catalog/src/error.rs | 28 + litellm-rust/crates/model-catalog/src/lib.rs | 16 + .../crates/model-catalog/src/model_info.rs | 665 ++++++++++++++++++ .../crates/model-catalog/src/schema.rs | 7 + .../crates/model-catalog/tests/catalog.rs | 253 +++++++ .../crates/model-catalog/tests/spec_parity.rs | 121 ++++ 11 files changed, 1440 insertions(+) create mode 100644 litellm-rust/crates/model-catalog/Cargo.toml create mode 100644 litellm-rust/crates/model-catalog/README.md create mode 100644 litellm-rust/crates/model-catalog/benches/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/src/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/src/error.rs create mode 100644 litellm-rust/crates/model-catalog/src/lib.rs create mode 100644 litellm-rust/crates/model-catalog/src/model_info.rs create mode 100644 litellm-rust/crates/model-catalog/src/schema.rs create mode 100644 litellm-rust/crates/model-catalog/tests/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/tests/spec_parity.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f3510000c44..4427057a3b9 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3054,6 +3054,20 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-model-catalog" +version = "0.1.0" +dependencies = [ + "criterion", + "indexmap 2.14.0", + "litellm-model-catalog", + "rstest", + "schemars 1.2.2", + "serde", + "serde_json", + "thiserror 2.0.19", +] + [[package]] name = "litellm-python-bridge" version = "0.1.0" @@ -4756,10 +4770,23 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "dyn-clone", "ref-cast", + "schemars_derive", "serde", "serde_json", ] +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4848,6 +4875,17 @@ dependencies = [ "syn 3.0.0", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.0", +] + [[package]] name = "serde_json" version = "1.0.150" diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml new file mode 100644 index 00000000000..ea75c6386d8 --- /dev/null +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "litellm-model-catalog" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +schema = ["dep:schemars"] + +[dependencies] +indexmap = { version = "2.14.0", features = ["serde"] } +schemars = { version = "1.0", optional = true } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +criterion.workspace = true +rstest.workspace = true +litellm-model-catalog = { path = ".", features = ["schema"] } + +[[bench]] +name = "catalog" +harness = false diff --git a/litellm-rust/crates/model-catalog/README.md b/litellm-rust/crates/model-catalog/README.md new file mode 100644 index 00000000000..973f7190614 --- /dev/null +++ b/litellm-rust/crates/model-catalog/README.md @@ -0,0 +1,25 @@ +# Model catalog + +`litellm-model-catalog` builds an immutable snapshot from caller supplied JSON bytes. It has no network, Python, registration, or refresh behavior. The caller supplies optional source, revision, and ETag provenance. Parse and validation are separate so small synthetic catalogs can use explicit integrity limits + +The parser treats `sample_spec` and `fallback_generalizations` as reserved top level metadata. `fallback_rules()` exposes the typed rule array when present; this crate does not execute regex generalizations. Model entries retain all JSON fields except `aliases`, including unknown fields. `field()` returns `None` for an absent key and a JSON null, false, or zero value for a present key. The returned values are borrowed, so callers cannot mutate the snapshot + +Each entry also deserializes into `ModelInfo`, a typed mirror of `model_prices_and_context_window.schema.json`'s `modelEntry` definition, reachable via `ModelEntry::info()`. All schema fields are optional on `ModelInfo`, including `litellm_provider` which the schema marks required, so small synthetic catalogs still parse. Unknown fields are not part of `ModelInfo`; they remain on `fields()`. Building with the `schema` feature adds `schemars` derives and exposes `model_entry_json_schema()` for emitting the entry's JSON Schema. Parse and validation failures are reported by the `Error` enum in `error.rs`, while catalog logic lives in `catalog.rs` + +The integration tests read the repository's catalog and schema files at test time, assert every entry round-trips through `ModelInfo`, and verify that the generated schema's properties match the repository schema + +Aliases point to their canonical entries. An alias that exactly matches any canonical key is skipped; the first canonical entry claiming an alias wins. Invalid alias lists and nonstring names are skipped and reported by `alias_issues()`. Exact lookup wins. For a case insensitive miss, the last key with the same lowercase spelling wins, following Python's lowercase map built after aliases are appended. This uses Rust Unicode lowercasing, which can differ from Python for unusual Unicode model IDs + +`validate()` counts canonical entries before alias expansion and excludes both reserved keys. It enforces an explicit minimum and backup shrink ratio, with Python defaults of 50 models and 0.5. Parsing rejects nonobject model entries and known fields with the wrong JSON type, but ignores unknown fields. It does not enforce every constraint in the JSON schema, calculate prices, resolve providers, or check provenance authenticity. The caller decides how to handle validation failures + +This snapshot does not represent Python's live mutable `litellm.model_cost`, nested dict and list mutation, or mutation of dicts previously returned by Python APIs. It has no bridge or runtime integration + +## Benchmarks + +`cargo bench -p litellm-model-catalog --bench catalog` measures parsing plus alias indexing and exact lookup. For a local Python baseline on the same fixture, use: + +```sh +python3 -m timeit -s 'import json, pathlib; body = pathlib.Path("../model_prices_and_context_window.json").read_bytes()' 'json.loads(body)' +``` + +Run these commands from `litellm-rust`. Python's command measures JSON loading only, without alias expansion or snapshot construction. The Rust benchmark does not include future Python object materialization, so these numbers are not an end to end runtime comparison diff --git a/litellm-rust/crates/model-catalog/benches/catalog.rs b/litellm-rust/crates/model-catalog/benches/catalog.rs new file mode 100644 index 00000000000..d1f51507c2b --- /dev/null +++ b/litellm-rust/crates/model-catalog/benches/catalog.rs @@ -0,0 +1,21 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use litellm_model_catalog::{Catalog, Provenance}; +use std::hint::black_box; + +fn benchmarks(c: &mut Criterion) { + let body = include_bytes!("../../../../model_prices_and_context_window.json"); + c.bench_function("parse_current_catalog", |b| { + b.iter(|| Catalog::parse(black_box(body), Provenance::default()).unwrap()) + }); + let catalog = Catalog::parse(body, Provenance::default()).unwrap(); + let key = catalog + .model_names() + .next() + .expect("catalog must have a benchmark key"); + c.bench_function("lookup_catalog_key", |b| { + b.iter(|| black_box(&catalog).lookup(black_box(key))) + }); +} + +criterion_group!(benches, benchmarks); +criterion_main!(benches); diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs new file mode 100644 index 00000000000..dc7564f9bee --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -0,0 +1,241 @@ +use crate::error::Error; +use crate::model_info::{FallbackGeneralizations, FallbackRule, ModelInfo}; +use indexmap::IndexMap; +use serde::Deserialize; +use serde_json::{Map, Value}; +use std::collections::HashMap; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Provenance { + pub source: Option, + pub revision: Option, + pub etag: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct IntegrityLimits { + pub backup_model_count: usize, + pub min_model_count: usize, + pub min_backup_ratio: f64, +} + +impl IntegrityLimits { + pub fn python_defaults(backup_model_count: usize) -> Self { + Self { + backup_model_count, + min_model_count: 50, + min_backup_ratio: 0.5, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AliasIssue { + InvalidList { model: String }, + InvalidName { model: String }, + CanonicalCollision { model: String, alias: String }, + AliasCollision { model: String, alias: String }, +} + +#[derive(Clone, Debug)] +pub struct ModelEntry { + fields: Map, + info: ModelInfo, +} + +impl ModelEntry { + pub fn field(&self, name: &str) -> Option<&Value> { + self.fields.get(name) + } + pub fn fields(&self) -> &Map { + &self.fields + } + /// The entry deserialized into the typed mirror of the catalog schema. + pub fn info(&self) -> &ModelInfo { + &self.info + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ModelMatch<'a> { + pub matched_key: &'a str, + pub canonical_key: &'a str, + pub entry: &'a ModelEntry, +} + +#[derive(Debug)] +pub struct Catalog { + entries: IndexMap, + aliases: IndexMap, + lowercase_keys: HashMap, + sample_spec: Option, + fallback_generalizations: Option, + provenance: Provenance, + alias_issues: Vec, +} + +impl Catalog { + pub fn parse(body: &[u8], provenance: Provenance) -> Result { + let root: IndexMap = serde_json::from_slice(body)?; + if root.is_empty() { + return Err(Error::Empty); + } + + let mut entries = IndexMap::with_capacity(root.len()); + let mut alias_lists = Vec::new(); + let mut alias_issues = Vec::new(); + let mut sample_spec = None; + let mut fallback_generalizations = None; + for (name, value) in root { + match name.as_str() { + "sample_spec" => { + sample_spec = Some(value); + continue; + } + "fallback_generalizations" => { + fallback_generalizations = + Some(serde_json::from_value::(value)?); + continue; + } + _ => {} + } + let Value::Object(ref object) = value else { + return Err(Error::EntryNotObject { model: name }); + }; + let info = ModelInfo::deserialize(object)?; + let Value::Object(mut fields) = value else { + unreachable!("value checked is_object above") + }; + if let Some(aliases) = fields.remove("aliases") + && !aliases.is_null() + { + match aliases { + Value::Array(names) => alias_lists.push((name.clone(), names)), + _ => alias_issues.push(AliasIssue::InvalidList { + model: name.clone(), + }), + } + } + entries.insert(name, ModelEntry { fields, info }); + } + + let mut aliases = IndexMap::new(); + for (model, names) in alias_lists { + for name in names { + let Value::String(alias) = name else { + alias_issues.push(AliasIssue::InvalidName { + model: model.clone(), + }); + continue; + }; + if entries.contains_key(&alias) { + alias_issues.push(AliasIssue::CanonicalCollision { + model: model.clone(), + alias, + }); + } else if aliases.contains_key(&alias) { + alias_issues.push(AliasIssue::AliasCollision { + model: model.clone(), + alias, + }); + } else { + aliases.insert(alias, model.clone()); + } + } + } + + let lowercase_keys = entries + .keys() + .chain(aliases.keys()) + .map(|key| (key.to_lowercase(), key.clone())) + .collect(); + Ok(Self { + entries, + aliases, + lowercase_keys, + sample_spec, + fallback_generalizations, + provenance, + alias_issues, + }) + } + + pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> { + if !limits.min_backup_ratio.is_finite() || !(0.0..=1.0).contains(&limits.min_backup_ratio) { + return Err(Error::InvalidRatio); + } + let actual = self.entries.len(); + if actual < limits.min_model_count { + return Err(Error::BelowMinimum { + actual, + minimum: limits.min_model_count, + }); + } + if limits.backup_model_count > 0 + && (actual as f64) < (limits.backup_model_count as f64) * limits.min_backup_ratio + { + return Err(Error::Shrunk { + actual, + backup: limits.backup_model_count, + ratio: limits.min_backup_ratio, + }); + } + Ok(()) + } + + pub fn lookup(&self, key: &str) -> Option> { + let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) { + key + } else { + self.lowercase_keys.get(&key.to_lowercase())?.as_str() + }; + let canonical_key = self + .aliases + .get(matched_key) + .map(String::as_str) + .unwrap_or(matched_key); + let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?; + let matched_key = self + .entries + .get_key_value(matched_key) + .map(|(key, _)| key.as_str()) + .or_else(|| { + self.aliases + .get_key_value(matched_key) + .map(|(key, _)| key.as_str()) + })?; + Some(ModelMatch { + matched_key, + canonical_key, + entry, + }) + } + + pub fn model_count(&self) -> usize { + self.entries.len() + } + pub fn model_names(&self) -> impl Iterator { + self.entries.keys().map(String::as_str) + } + pub fn alias_count(&self) -> usize { + self.aliases.len() + } + pub fn aliases(&self) -> &IndexMap { + &self.aliases + } + pub fn alias_issues(&self) -> &[AliasIssue] { + &self.alias_issues + } + pub fn sample_spec(&self) -> Option<&Value> { + self.sample_spec.as_ref() + } + pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> { + self.fallback_generalizations.as_ref() + } + pub fn fallback_rules(&self) -> Option<&[FallbackRule]> { + Some(self.fallback_generalizations.as_ref()?.rules.as_slice()) + } + pub fn provenance(&self) -> &Provenance { + &self.provenance + } +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs new file mode 100644 index 00000000000..83617312fff --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -0,0 +1,28 @@ +use thiserror::Error; + +/// Failures from parsing or validating a catalog snapshot. +#[derive(Debug, Error)] +pub enum Error { + /// The body is not valid JSON, or a model entry fails typed deserialization. + #[error("invalid JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog has no entries at all. + #[error("catalog is empty")] + Empty, + /// A non-reserved top level value is not a JSON object. + #[error("model {model:?} must be an object")] + EntryNotObject { model: String }, + /// Canonical entry count is under the configured minimum. + #[error("catalog has {actual} models, below minimum {minimum}")] + BelowMinimum { actual: usize, minimum: usize }, + /// Canonical entry count is under the configured backup shrink ratio. + #[error("catalog has {actual} models, below {ratio} of backup count {backup}")] + Shrunk { + actual: usize, + backup: usize, + ratio: f64, + }, + /// The configured minimum backup ratio is not finite or outside `[0, 1]`. + #[error("minimum backup ratio must be finite and between zero and one")] + InvalidRatio, +} diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs new file mode 100644 index 00000000000..9c942a5521c --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -0,0 +1,16 @@ +mod catalog; +mod error; +mod model_info; +#[cfg(feature = "schema")] +mod schema; + +pub use catalog::{AliasIssue, Catalog, IntegrityLimits, ModelEntry, ModelMatch, Provenance}; +pub use error::Error; +pub use model_info::{ + AudioFormat, FallbackGeneralizations, FallbackRule, InputModality, Mode, ModelInfo, + OffPeakPricing, OffPeakWindow, OutputModality, ReasoningEffort, SearchContextCostPerQuery, + TieredRate, UtcHours, VertexAiAudioApi, WebSearchBillingUnit, Weekday, +}; + +#[cfg(feature = "schema")] +pub use schema::model_entry_json_schema; diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs new file mode 100644 index 00000000000..77a8b768e38 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -0,0 +1,665 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Primary API surface / task type of the model. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum Mode { + AudioSpeech, + AudioTranscription, + Chat, + Completion, + Embedding, + Evaluation, + Guardrail, + ImageEdit, + ImageGeneration, + Moderation, + Ocr, + Realtime, + Rerank, + Responses, + Search, + VectorStore, + VideoGeneration, +} + +/// Reasoning effort level accepted or applied by the model. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, + Xhigh, + Max, +} + +/// Gemini audio generation API the model is served through. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum VertexAiAudioApi { + LyriaPredict, + LyriaInteractions, +} + +/// Whether web search is billed per query or per prompt. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum WebSearchBillingUnit { + PerQuery, + PerPrompt, +} + +/// Audio container format the model can return. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + Mp3, + Wav, +} + +/// Input modality the model accepts. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum InputModality { + Text, + Image, + Audio, + Video, +} + +/// Output modality the model can produce. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum OutputModality { + Text, + Image, + Audio, + Video, + Code, +} + +/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum UtcHours { + Single(String), + Multiple(Vec), +} + +/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum Weekday { + Number(u8), + Name(String), +} + +/// One off-peak window entry inside `windows`. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct OffPeakWindow { + pub hours_utc: UtcHours, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub weekdays: Option>, +} + +/// Rates that replace the same-named base fields inside the stated UTC windows. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct OffPeakPricing { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hours_utc: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub windows: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub weekday_timezone: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, +} + +/// USD cost per web search query, keyed by search context size. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct SearchContextCostPerQuery { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_low: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_medium: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_high: Option, +} + +/// One tier of a context-length or result-count tiered rate. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct TieredRate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range: Option<[f64; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_results_range: Option<[f64; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_query: Option, +} + +/// One regex rule generalizing unknown model ids to known families. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct FallbackRule { + pub name: String, + pub pattern: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(flatten)] + pub extra: BTreeMap, +} + +/// Regex rules that generalize unknown model ids to known families; not a model entry. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct FallbackGeneralizations { + pub rules: Vec, +} + +/// Typed mirror of one catalog model entry. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ModelInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub annotation_cost_per_page: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub annotation_cost_per_page_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_transcription_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bedrock_converse_supports_strict_tools: Option, + /// Highest reasoning effort the Bedrock output_config accepts for this model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bedrock_output_config_effort_ceiling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_audio_token_cost: Option, + /// USD per token written to the provider's prompt cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_1hr: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_200k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_audio_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_image_token_cost: Option, + /// USD per prompt token served from the provider's prompt cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_512k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub citation_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_interpreter_cost_per_session: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub comment: Option, + /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_reasoning_effort: Option, + /// Date the provider deprecates the model, YYYY-MM-DD. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deprecation_date: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gemini_audio_only_live: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gemini_native_audio: Option, + /// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub google_maps_grounding_cost_per_query: Option, + /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub guardrail_cost_per_unit: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_per_second: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_per_second_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token_batches: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_character: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_character_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_pixel: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_query: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_request: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_second: Option, + /// USD per prompt token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_512k_tokens: Option, + /// USD per prompt token via the provider's batch API. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_cache_hit: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_15s_interval: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_8s_interval: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_dbu_cost_per_token: Option, + /// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub litellm_provider: Option, + /// Maximum prompt/context tokens the model accepts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_input_tokens: Option, + /// Maximum tokens the model can generate in one response. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + /// Legacy field: max output tokens if the provider specifies it, else max input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + /// Free-form notes about the entry (e.g. pricing derivation). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + /// Primary API surface / task type of the model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_credit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_page: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_page_batches: Option, + /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub off_peak_pricing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_audio_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_character: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_character_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_1024: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_1536: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_512: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_pixel: Option, + /// USD per reasoning/thinking token, when billed separately. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_1080p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_2k: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_480p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_4k: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_720p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_768p: Option, + /// USD per generated token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_512k_tokens: Option, + /// USD per generated token via the provider's batch API. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_video_per_second: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_video_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_dbu_cost_per_token: Option, + /// Embedding dimension for embedding models. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_vector_size: Option, + /// Smallest prefix the provider will actually cache; absent means the provider default applies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_cache_min_tokens: Option, + /// Provider-internal routing hints (e.g. bedrock_invocation_schema). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_entry: Option>, + /// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_effort_levels: Option>, + /// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_endpoint_uplift_multiplier: Option, + /// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_processing_uplift_multiplier_eu: Option, + /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_processing_uplift_multiplier_us: Option, + /// Provider default requests-per-minute limit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rpm: Option, + /// USD cost per web search query, keyed by search context size. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_cost_per_query: Option, + /// URL of the provider pricing/model page this entry was taken from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Audio container formats the model can return. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_audio_formats: Option>, + /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_endpoints: Option>, + /// Input modalities the model accepts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_modalities: Option>, + /// Output modalities the model can produce. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_output_modalities: Option>, + /// Cloud regions the model is available in ('global' or region ids). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_regions: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_adaptive_thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_anthropic_compaction: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_anthropic_thinking_payload: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_assistant_prefill: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_audio_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_audio_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_computer_use: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_embedding_image_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_fast_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_forced_tool_use: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_function_calling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_image_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_image_size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_legacy_thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_low_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_max_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_mid_conversation_system: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_minimal_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_multimodal: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_native_streaming: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_native_structured_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_none_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_nova_canvas_image_edit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_output_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_parallel_function_calling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_parallel_tool_use_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_pdf_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_prompt_cache_breakpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_prompt_caching: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_reasoning: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_response_schema: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_sampling_params: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_speed: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_system_messages: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_thinking_cache_preservation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tool_choice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tool_search: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_url_context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_video_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_vision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_web_search: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_xhigh_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_always_on: Option, + /// Context-length or result-count tiered rates; each tier's costs apply within its range. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tiered_pricing: Option>, + /// Provider default tokens-per-minute limit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tpm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub use_openai_responses_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub uses_embed_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vertex_ai_audio_api: Option, + /// Whether web search is billed per query or per prompt. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub web_search_billing_unit: Option, +} diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs new file mode 100644 index 00000000000..82cfd6c0352 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -0,0 +1,7 @@ +use crate::model_info::ModelInfo; + +/// JSON Schema for one catalog model entry, mirroring +/// `model_prices_and_context_window.schema.json`'s `modelEntry` definition. +pub fn model_entry_json_schema() -> schemars::Schema { + schemars::schema_for!(ModelInfo) +} diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs new file mode 100644 index 00000000000..bcadd38e908 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -0,0 +1,253 @@ +use std::path::{Path, PathBuf}; + +use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance}; +use rstest::{fixture, rstest}; +use serde_json::json; + +const ALPHA_FIXTURE: &[u8] = br#"{ + "sample_spec":{"explanation":"example"}, + "fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}, + "Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false, + "optional":null,"unknown":{"nested":[1,{"x":true}]}} +}"#; + +#[fixture] +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +#[fixture] +fn current_catalog(repo_root: PathBuf) -> Catalog { + let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); + Catalog::parse(&body, Provenance::default()).unwrap() +} + +#[fixture] +fn backup_catalog(repo_root: PathBuf) -> Catalog { + let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json")) + .unwrap(); + Catalog::parse(&body, Provenance::default()).unwrap() +} + +#[fixture] +fn fixture_catalog() -> Catalog { + Catalog::parse( + ALPHA_FIXTURE, + Provenance { + source: Some("fixture".into()), + revision: Some("rev".into()), + etag: None, + }, + ) + .unwrap() +} + +#[rstest] +fn preserves_fields_and_metadata(fixture_catalog: Catalog) { + let catalog = fixture_catalog; + let entry = catalog.lookup("SHORT").unwrap(); + assert_eq!(entry.canonical_key, "Alpha"); + assert_eq!(entry.matched_key, "short"); + assert_eq!(entry.entry.field("price"), Some(&json!(0))); + assert_eq!(entry.entry.field("enabled"), Some(&json!(false))); + assert_eq!(entry.entry.field("optional"), Some(&json!(null))); + assert_eq!(entry.entry.field("missing"), None); + assert_eq!( + entry.entry.field("unknown"), + Some(&json!({"nested":[1,{"x":true}]})) + ); + assert_eq!(entry.entry.field("aliases"), None); + assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test")); + assert_eq!( + catalog.sample_spec(), + Some(&json!({"explanation":"example"})) + ); + assert_eq!(catalog.fallback_rules().unwrap().len(), 1); + assert_eq!(catalog.provenance().revision.as_deref(), Some("rev")); + assert_eq!(catalog.model_count(), 1); +} + +#[rstest] +fn snapshot_does_not_borrow_source() { + let mut source = ALPHA_FIXTURE.to_vec(); + let catalog = Catalog::parse(&source, Provenance::default()).unwrap(); + source.fill(b' '); + + let entry = catalog.lookup("short").unwrap(); + assert_eq!(entry.canonical_key, "Alpha"); + assert_eq!(entry.entry.field("price"), Some(&json!(0))); +} + +#[rstest] +#[case("Shared", "First")] +#[case("Second", "Second")] +#[case("shared", "Second")] +#[case("FIRST", "First")] +#[case("sHaReD", "Second")] +fn alias_collisions_and_case_fallback_follow_python_order( + #[case] lookup: &str, + #[case] expected: &str, +) { + let catalog = Catalog::parse( + br#"{ + "First":{"aliases":["Shared","Second","first"],"value":1}, + "Second":{"aliases":["Shared","sHaReD"],"value":2}, + "SHARED":{"value":3} + }"#, + Provenance::default(), + ) + .unwrap(); + assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected); + assert_eq!(catalog.alias_count(), 3); + assert!( + catalog + .alias_issues() + .contains(&AliasIssue::CanonicalCollision { + model: "First".into(), + alias: "Second".into(), + }) + ); + assert!( + catalog + .alias_issues() + .contains(&AliasIssue::AliasCollision { + model: "Second".into(), + alias: "Shared".into(), + }) + ); +} + +#[derive(Debug)] +enum ValidationOutcome { + Ok, + Shrunk, + BelowMinimum, + InvalidRatio, +} + +#[rstest] +#[case( + IntegrityLimits { + backup_model_count: 2, + min_model_count: 1, + min_backup_ratio: 0.5, + }, + ValidationOutcome::Ok +)] +#[case( + IntegrityLimits { + backup_model_count: 3, + min_model_count: 1, + min_backup_ratio: 0.5, + }, + ValidationOutcome::Shrunk +)] +#[case( + IntegrityLimits { + backup_model_count: 0, + min_model_count: 2, + min_backup_ratio: 0.5, + }, + ValidationOutcome::BelowMinimum +)] +#[case( + IntegrityLimits { + backup_model_count: 0, + min_model_count: 0, + min_backup_ratio: f64::NAN, + }, + ValidationOutcome::InvalidRatio +)] +fn integrity_uses_canonical_count_and_strict_shrink_boundary( + #[case] limits: IntegrityLimits, + #[case] expected: ValidationOutcome, +) { + let catalog = Catalog::parse( + br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#, + Provenance::default(), + ) + .unwrap(); + let actual = catalog.validate(limits); + match expected { + ValidationOutcome::Ok => assert!(actual.is_ok()), + ValidationOutcome::Shrunk => { + assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. }))) + } + ValidationOutcome::BelowMinimum => { + assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. }))) + } + ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))), + } +} + +#[derive(Debug)] +enum MalformedOutcome { + Empty, + Json, + EntryNotObject, +} + +#[rstest] +#[case::empty(b"{}", MalformedOutcome::Empty)] +#[case::invalid_json(b"{", MalformedOutcome::Json)] +#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)] +#[case::fallback_rules_missing( + br#"{"fallback_generalizations":{},"a":{}}"#, + MalformedOutcome::Json +)] +fn malformed_input_and_aliases_have_typed_outcomes( + #[case] body: &[u8], + #[case] expected: MalformedOutcome, +) { + let actual = Catalog::parse(body, Provenance::default()); + match expected { + MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))), + MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))), + MalformedOutcome::EntryNotObject => { + assert!(matches!(actual, Err(Error::EntryNotObject { .. }))) + } + } +} + +#[rstest] +fn invalid_aliases_are_reported_not_fatal() { + let catalog = Catalog::parse( + br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#, + Provenance::default(), + ) + .unwrap(); + assert_eq!( + catalog.alias_issues(), + &[ + AliasIssue::InvalidList { model: "a".into() }, + AliasIssue::InvalidName { model: "b".into() }, + ] + ); + assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b"); + assert!(catalog.lookup("missing").is_none()); +} + +#[rstest] +fn parses_current_and_packaged_catalogs_without_pinning_counts( + current_catalog: Catalog, + backup_catalog: Catalog, +) { + assert!(current_catalog.model_count() > 0); + assert!(backup_catalog.model_count() > 0); + assert!(current_catalog.sample_spec().is_some()); + assert!(backup_catalog.sample_spec().is_some()); + assert!( + current_catalog + .validate(IntegrityLimits::python_defaults( + backup_catalog.model_count() + )) + .is_ok() + ); + for name in current_catalog.model_names() { + let entry = current_catalog.lookup(name).unwrap().entry; + assert_eq!( + entry.info().litellm_provider.is_some(), + entry.field("litellm_provider").is_some() + ); + } +} diff --git a/litellm-rust/crates/model-catalog/tests/spec_parity.rs b/litellm-rust/crates/model-catalog/tests/spec_parity.rs new file mode 100644 index 00000000000..7296d96f798 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/spec_parity.rs @@ -0,0 +1,121 @@ +use std::collections::{BTreeSet, HashSet}; +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use litellm_model_catalog::{ + Catalog, FallbackGeneralizations, ModelInfo, Provenance, model_entry_json_schema, +}; +use rstest::{fixture, rstest}; +use serde_json::{Map, Value}; + +#[fixture] +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn json_eq(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(), + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left + .iter() + .all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other))) + } + _ => left == right, + } +} + +fn keys(value: &Map) -> BTreeSet { + value.keys().cloned().collect() +} + +fn symmetric_difference(left: &BTreeSet, right: &BTreeSet) -> BTreeSet { + left.symmetric_difference(right).cloned().collect() +} + +#[rstest] +#[case("model_prices_and_context_window.json")] +#[case("litellm/model_prices_and_context_window_backup.json")] +fn every_entry_round_trips_through_model_info(repo_root: PathBuf, #[case] filename: &str) { + let body = std::fs::read(repo_root.join(filename)).unwrap(); + let document: IndexMap = serde_json::from_slice(&body).unwrap(); + for (model_name, value) in document { + if matches!( + model_name.as_str(), + "sample_spec" | "fallback_generalizations" + ) { + continue; + } + let object = value + .as_object() + .unwrap_or_else(|| panic!("{model_name} is not an object")); + let info: ModelInfo = serde_json::from_value(value.clone()) + .unwrap_or_else(|error| panic!("{model_name} does not deserialize: {error}")); + let serialized = serde_json::to_value(info).unwrap(); + let serialized_object = serialized + .as_object() + .unwrap_or_else(|| panic!("{model_name} did not serialize as an object")); + let mut expected = object.clone(); + expected.remove("aliases"); + let expected_keys = keys(&expected); + let serialized_keys = keys(serialized_object); + assert_eq!( + expected_keys, + serialized_keys, + "{model_name} key difference: {:?}", + symmetric_difference(&expected_keys, &serialized_keys) + ); + assert!( + json_eq(&Value::Object(expected), &serialized), + "{model_name} changed during ModelInfo round-trip" + ); + } +} + +#[rstest] +fn fallback_generalizations_are_typed(repo_root: PathBuf) { + let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); + let document: Map = serde_json::from_slice(&body).unwrap(); + let Some(raw_rules) = document.get("fallback_generalizations") else { + return; + }; + let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap(); + let catalog = Catalog::parse(&body, Provenance::default()).unwrap(); + assert!( + catalog + .fallback_rules() + .is_some_and(|rules| !rules.is_empty()) + ); +} + +#[rstest] +fn generated_schema_properties_match_repo_schema(repo_root: PathBuf) { + let body = + std::fs::read(repo_root.join("model_prices_and_context_window.schema.json")).unwrap(); + let document: Value = serde_json::from_slice(&body).unwrap(); + let repo_entry_properties = document["$defs"]["modelEntry"]["properties"] + .as_object() + .unwrap(); + let generated = serde_json::to_value(model_entry_json_schema()).unwrap(); + let generated_properties = generated["properties"].as_object().unwrap(); + let expected = keys(repo_entry_properties); + let actual = keys(generated_properties); + assert_eq!( + expected, + actual, + "modelEntry property difference: {:?}", + symmetric_difference(&expected, &actual) + ); + + let repo_root_properties = document["properties"].as_object().unwrap(); + let actual_root: HashSet = repo_root_properties.keys().cloned().collect(); + let expected_root: HashSet = ["sample_spec", "fallback_generalizations"] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(actual_root, expected_root); +} From da82ea8e94b737b7f5873609edf1e842699e5aa2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:30:29 -0700 Subject: [PATCH 17/17] fix(ui): let the Create Key user picker find users by user_id, not just email (#41687) * feat(ui): search users by id or email when assigning a key owner Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): label users without an email by user id in key owner picker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): freeze merged user-filter where, format create key test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): suppress module-global patch findings in ui_view_users search test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep merged user-filter where as a plain dict for prisma serialization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): forward search param from userFilterUICall to /user/filter/ui Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): mention user ID in the Create Key user picker helper text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: jesus Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../internal_user_endpoints.py | 14 +++- .../test_internal_user_endpoints.py | 66 +++++++++++++++++++ .../src/components/networking.test.ts | 23 +++++++ .../src/components/networking.tsx | 1 + .../create_key_button.integration.test.tsx | 53 ++++++++++----- .../organisms/create_key_button.tsx | 10 +-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 7 files changed, 148 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 133181e9203..587ae416096 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2741,6 +2741,10 @@ async def _resolve_team_org_filter( async def ui_view_users( user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"), user_email: str | None = fastapi.Query(default=None, description="User email in the request parameters"), + search: str | None = fastapi.Query( + default=None, + description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).", + ), team_id: str | None = fastapi.Query( default=None, description="Team ID — used when a team admin searches for users to add to their team", @@ -2750,7 +2754,7 @@ async def ui_view_users( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Filter users based on partial match of user_id or email with pagination. + Filter users based on partial match of user_id or email, or combined ``search``, with pagination. Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag (stored in the ``litellm_uisettings`` table): @@ -2802,9 +2806,15 @@ async def ui_view_users( if org_filter_ids is not None: where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}} + where: Final[Mapping[str, object]] = { # mutable-ok: prisma serializes `where`, keep it a plain dict + key: value + for key, value in (*where_conditions.items(), *_user_search_where(search).items()) + if value is not None + } + # Query users with pagination and filters users: Final = await _user_table(prisma_client).find_many( - where=where_conditions, + where=where, skip=skip, take=page_size, order={"created_at": "desc"}, diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 2d1049b143e..0465572235b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -2,6 +2,7 @@ import asyncio import hashlib import json import logging +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from types import SimpleNamespace from typing import Final @@ -37,6 +38,7 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( ui_view_users, ) from litellm.proxy.proxy_server import app +from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains from tests.test_litellm.proxy.management_endpoints.jwt_key_mapping_doubles import ( CascadingJWTMappingTable, JWTMappingRow, @@ -119,6 +121,70 @@ async def test_ui_view_users_proxy_admin_no_org_filter(mocker): ) +UserWhereCondition = InsensitiveContains | Sequence[Mapping[str, InsensitiveContains]] + + +def _matches_user_where(row: LiteLLM_UserTableFiltered, where: Mapping[str, UserWhereCondition]) -> bool: + def matches(field: str, condition: UserWhereCondition) -> bool: + if not isinstance(condition, Mapping): + return any(_matches_user_where(row, branch) for branch in condition) + value: Final = {"user_id": row.user_id, "user_email": row.user_email}[field] + return value is not None and condition["contains"].lower() in value.lower() + + return all(matches(field, condition) for field, condition in where.items()) + + +@pytest.mark.parametrize( + "params, expected_user_ids", + [ + ({"search": "SVC"}, ["svc-bot"]), + ({"search": "ali"}, ["alice-admin"]), + ({"search": "example.com"}, ["alice-admin"]), + ({"search": "admin"}, ["alice-admin"]), + ({"user_email": "svc"}, []), + ({"user_id": "svc"}, ["svc-bot"]), + ({"search": "ali", "user_id": "svc"}, []), + ], +) +def test_ui_view_users_search_matches_user_id_or_email( + mocker: MockerFixture, params: Mapping[str, str], expected_user_ids: list[str] +): + """ + search= returns users whose user_id or user_email contains the value (case-insensitive), + including users with no email; user_id=/user_email= keep filtering a single field and AND with search. + """ + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + users = ( + LiteLLM_UserTableFiltered(user_id="alice-admin", user_email="alice@example.com"), + LiteLLM_UserTableFiltered(user_id="svc-bot", user_email=None), + LiteLLM_UserTableFiltered(user_id="bob", user_email="bob@corp.io"), + ) + + async def mock_find_many(*, where: Mapping[str, UserWhereCondition], **_: object): + return [user for user in users if _matches_user_where(user, where)] + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_many = mock_find_many + mocker.patch( # test-quality-ok: endpoint reads settings via module global; same seam as sibling tests + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={}, + ) + mocker.patch( # test-quality-ok: endpoint reads prisma_client via module global; same seam as sibling tests + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + response = client.get("/user/filter/ui", params=params) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert [user["user_id"] for user in response.json()] == expected_user_ids + + @pytest.mark.asyncio async def test_ui_view_users_org_admin_filtered_by_org(mocker): """ diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 3b2a17101ee..e14f1939ee1 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -913,3 +913,26 @@ describe("fetchMemoryList search serialization", () => { expect(lastParams(mockFetch).has("search")).toBe(false); }); }); + +describe("userFilterUICall", () => { + let currentFetch: typeof global.fetch; + + beforeEach(() => { + currentFetch = global.fetch; + }); + + afterEach(() => { + global.fetch = currentFetch; + }); + + it("forwards the search param to /user/filter/ui", async () => { + const mockFetch = vi.fn().mockResolvedValue({ ok: true, text: async () => "[]" } as any); + global.fetch = mockFetch as any; + + await Networking.userFilterUICall("sk-test", new URLSearchParams({ search: "svc" })); + + const parsed = new URL(mockFetch.mock.calls[0][0] as string, "http://localhost"); + expect(parsed.pathname).toContain("/user/filter/ui"); + expect(parsed.searchParams.get("search")).toBe("svc"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d358d23408c..76c6a3cb935 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1983,6 +1983,7 @@ export const userFilterUICall = async (accessToken: string, params: URLSearchPar user_email: params.get("user_email") || undefined, user_id: params.get("user_id") || undefined, team_id: params.get("team_id") || undefined, + search: params.get("search") || undefined, }, }); } catch (error) { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index a6b37ac26ff..0bf3268379c 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -204,7 +204,8 @@ const openModal = async (props: Partial> return view; }; -const userSearchInput = (): Promise => screen.findByPlaceholderText("Type email to search for users"); +const userSearchInput = (): Promise => + screen.findByPlaceholderText("Type email or user ID to search for users"); const openSection = async (name: RegExp) => { await userEvent.click(await screen.findByRole("button", { name })); @@ -619,7 +620,7 @@ describe("CreateKey", () => { it("mounts the user search control only once Another User is chosen", async () => { await openModal(); - expect(screen.queryByPlaceholderText("Type email to search for users")).not.toBeInTheDocument(); + expect(screen.queryByPlaceholderText("Type email or user ID to search for users")).not.toBeInTheDocument(); await userEvent.click(screen.getByRole("radio", { name: "Another User" })); @@ -946,18 +947,18 @@ describe("CreateKey", () => { expect(vi.mocked(userFilterUICall)).toHaveBeenCalledTimes(1); const params = vi.mocked(userFilterUICall).mock.calls[0][1] as URLSearchParams; - expect(params.get("user_email")).toBe("alice"); + expect(params.get("search")).toBe("alice"); } finally { vi.useRealTimers(); } }); it("keeps the current search's users when an abandoned search answers last", async () => { - const answers = new Map void>(); + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -984,12 +985,36 @@ describe("CreateKey", () => { expect(screen.getByRole("option", { name: "alice.smith@example.com (u-smith)" })).toBeInTheDocument(); }); - it("stops searching once the box is cleared and the abandoned search answers", async () => { - const answers = new Map void>(); + it("labels a user with no email by their user id", async () => { + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); + }) as never, + ); + + const user = userEvent.setup(); + renderCreateKey({ autoOpenCreate: true, prefillData: { owned_by: "another_user" } }); + const search = await userSearchInput(); + + await user.type(search, "svc"); + await waitFor(() => expect(answers.has("svc")).toBe(true), { timeout: 3000 }); + + await act(async () => { + answers.get("svc")?.([{ user_id: "svc-bot", user_email: null }]); + }); + + expect(await screen.findByRole("option", { name: "svc-bot" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /null/ })).not.toBeInTheDocument(); + }); + + it("stops searching once the box is cleared and the abandoned search answers", async () => { + const answers = new Map void>(); + vi.mocked(userFilterUICall).mockImplementation( + (_accessToken, params) => + new Promise((resolve) => { + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -1013,11 +1038,11 @@ describe("CreateKey", () => { }); it("keeps searching while a newer search is still in flight", async () => { - const answers = new Map void>(); + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -1047,12 +1072,12 @@ describe("CreateKey", () => { it("only warns about a failed search when it is the one the box is waiting on", async () => { const answers = new Map< string, - { resolve: (users: { user_id: string; user_email: string }[]) => void; reject: (error: Error) => void } + { resolve: (users: { user_id: string; user_email: string | null }[]) => void; reject: (error: Error) => void } >(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve, reject) => { - answers.set(params.get("user_email") ?? "", { resolve, reject }); + answers.set(params.get("search") ?? "", { resolve, reject }); }) as never, ); @@ -1099,9 +1124,7 @@ describe("CreateKey", () => { ]; vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => - Promise.resolve( - directory.filter((entry) => entry.user_email.includes(params.get("user_email") ?? "")), - ) as never, + Promise.resolve(directory.filter((entry) => entry.user_email.includes(params.get("search") ?? ""))) as never, ); const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 3127eab249e..25b986e4c9e 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -161,7 +161,7 @@ interface CreateKeyProps { interface User { user_id: string; - user_email: string; + user_email: string | null; role?: string; } @@ -570,7 +570,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp setUserSearchLoading(true); try { const params = new URLSearchParams(); - params.append("user_email", searchText); // Always search by email + params.append("search", searchText); if (accessToken == null) { return; } @@ -579,7 +579,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const data: User[] = response; const options: SearchSelectOption[] = data.map((user) => ({ - label: `${user.user_email} (${user.user_id})`, + label: user.user_email ? `${user.user_email} (${user.user_id})` : user.user_id, value: user.user_id, })); @@ -729,7 +729,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp onValueChange={control.onChange} onSearchChange={fetchUsers} isLoading={userSearchLoading} - placeholder="Type email to search for users" + placeholder="Type email or user ID to search for users" emptyText="No users found" loadingText="Searching..." inputId={control.id} @@ -741,7 +741,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Create User -
Search by email to find users
+
Search by email or user ID to find users
)} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be1e000bd66..df294a20e19 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17325,7 +17325,7 @@ export interface paths { }; /** * Ui View Users - * @description Filter users based on partial match of user_id or email with pagination. + * @description Filter users based on partial match of user_id or email, or combined ``search``, with pagination. * * Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag * (stored in the ``litellm_uisettings`` table): @@ -64338,6 +64338,8 @@ export interface operations { user_id?: string | null; /** @description User email in the request parameters */ user_email?: string | null; + /** @description Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive). */ + search?: string | null; /** @description Team ID — used when a team admin searches for users to add to their team */ team_id?: string | null; /** @description Page number for pagination */