scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'litellm_internal_staging' of https://github.com/BerriAI/litellm into litellm_gemini_maps_grounding_cost

Browse source 
# Conflicts:
#	type-discipline-budget.json
This commit is contained in:
mateo-berri 2026-08-26 15:33:28 -07:00
commit 241daa4cb7
36 changed files with 1586 additions and 483 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
litellm/cost_calculator.py
View file

@ -794,14 +794,27 @@ def _select_model_name_for_cost_calc(
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
if region_name is not None:
return_model = f"{custom_llm_provider}/{region_name}/{return_model}"
else:
return_model = f"{custom_llm_provider}/{return_model}"
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
return return_model
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069)."""
segments: Final = model.split("/")
if "/".join(segments[1:]) in litellm.model_cost:
return model
head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1
head: Final = "/".join(segments[:head_len])
tail: Final = segments[head_len:]
strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail))
candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1))
return next((candidate for candidate in candidates if candidate in litellm.model_cost), model)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _model_contains_known_llm_provider(model: str) -> bool:
"""

6
litellm/integrations/dotprompt/__init__.py
View file

@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
from .prompt_manager import strip_version_suffix
registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id
try:
dot_prompt_manager: Final = DotpromptManager(
prompt_directory=prompt_directory,
prompt_data=prompt_data,
prompt_file=prompt_file,
prompt_id=prompt_id,
prompt_id=registration_prompt_id,
)
return dot_prompt_manager

2
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement):
if prompt_id is None:
return False
try:
return prompt_id in self.prompt_manager.list_prompts()
return self.prompt_manager.get_prompt(prompt_id) is not None
except Exception:
# If there's any error accessing prompts, don't run prompt management
return False

31
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
return base
return None
class PromptTemplate:
"""Represents a single prompt template with metadata and content."""
@ -124,11 +131,13 @@ class PromptManager:
"content": "template content",
"metadata": {"model": "gpt-4", "temperature": 0.7, ...}
} + prompt_id
"""
if prompt_id:
prompt_data = {prompt_id: prompt_data}
for prompt_id, prompt_info in prompt_data.items():
A dict carrying a "content" key is a single flat template registered under
prompt_id; anything else is treated as already keyed by template ID.
"""
keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data
for template_id, prompt_info in keyed_prompts.items():
try:
content = prompt_info.get("content", "")
metadata = prompt_info.get("metadata", {})
@ -136,11 +145,11 @@ class PromptManager:
template = PromptTemplate(
content=content,
metadata=metadata,
template_id=prompt_id,
template_id=template_id,
)
self.prompts[prompt_id] = template
self.prompts[template_id] = template
except Exception:
# Optional: print(f"Error loading prompt from JSON: {prompt_id}")
# Optional: print(f"Error loading prompt from JSON: {template_id}")
pass
def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:
@ -272,8 +281,12 @@ class PromptManager:
if versioned_id in self.prompts:
return self.prompts[versioned_id]
# Fall back to base prompt_id
return self.prompts.get(prompt_id)
direct_match: Final = self.prompts.get(prompt_id)
if direct_match is not None:
return direct_match
base_prompt_id: Final = strip_version_suffix(prompt_id)
return self.prompts.get(base_prompt_id) if base_prompt_id else None
def list_prompts(self) -> list[str]:
"""Get a list of all available prompt IDs."""

140
litellm/model_prices_and_context_window_backup.json
View file

@ -1428,7 +1428,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"global.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
@ -1465,7 +1465,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"us.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
@ -1502,7 +1502,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"eu.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
@ -1539,7 +1539,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2933,7 +2933,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-5": {
"deprecation_date": "2026-10-19",
@ -2956,7 +2957,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-6": {
"deprecation_date": "2027-02-02",
@ -2987,7 +2989,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-7": {
"deprecation_date": "2027-04-06",
@ -3018,7 +3021,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 2048
},
"azure_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
@ -3050,7 +3054,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"azure_ai/claude-opus-5": {
"supports_mid_conversation_system": true,
@ -3113,7 +3118,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-opus-4-1": {
"deprecation_date": "2026-08-05",
@ -3135,7 +3141,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-4-5": {
"deprecation_date": "2026-10-19",
@ -3157,7 +3164,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
@ -3188,7 +3196,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-4-6": {
"deprecation_date": "2027-02-10",
@ -3214,7 +3223,8 @@
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 1024
},
"azure/computer-use-preview": {
"input_cost_per_token": 3e-06,
@ -14724,7 +14734,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4": {
"cache_creation_input_token_cost": 1.874999e-05,
@ -14746,7 +14757,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-opus-4-1": {
"cache_creation_input_token_cost": 1.874999e-05,
@ -14768,7 +14780,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-opus-4-5": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14791,7 +14804,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14814,7 +14828,8 @@
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4-7": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14916,7 +14931,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-4-1": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -14960,7 +14976,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -14983,7 +15000,8 @@
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-5": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -33899,7 +33917,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-opus-4.1": {
"input_cost_per_image": 0.0048,
@ -33919,7 +33938,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-sonnet-4": {
"input_cost_per_image": 0.0048,
@ -33942,7 +33962,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-sonnet-4.6": {
"supports_adaptive_thinking": true,
@ -33968,7 +33989,8 @@
"supports_reasoning": true,
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
@ -33987,7 +34009,8 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
@ -34008,7 +34031,8 @@
"supports_reasoning": true,
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-sonnet-4.5": {
"input_cost_per_image": 0.0048,
@ -34031,7 +34055,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
@ -34049,7 +34074,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-opus-4.7": {
"supports_adaptive_thinking": true,
@ -34072,7 +34098,8 @@
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"prompt_cache_min_tokens": 2048
},
"openrouter/anthropic/claude-opus-5": {
"prompt_cache_min_tokens": 512,
@ -36539,7 +36566,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 4096
},
"replicate/ibm-granite/granite-3.3-8b-instruct": {
"input_cost_per_token": 3e-08,
@ -36621,7 +36649,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 1024
},
"replicate/deepseek-ai/deepseek-v3": {
"input_cost_per_token": 1.45e-06,
@ -36696,7 +36725,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 1024
},
"replicate/openai/gpt-4.1": {
"input_cost_per_token": 2e-06,
@ -39725,7 +39755,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-opus-4": {
"cache_creation_input_token_cost": 1.875e-05,
@ -39744,7 +39775,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-opus-4.1": {
"cache_creation_input_token_cost": 1.875e-05,
@ -39763,7 +39795,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
@ -39783,7 +39816,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
@ -39805,7 +39839,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-sonnet-4": {
"cache_creation_input_token_cost": 3.75e-06,
@ -39824,7 +39859,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-sonnet-4.5": {
"cache_creation_input_token_cost": 3.75e-06,
@ -39842,7 +39878,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/cohere/command-a": {
"input_cost_per_token": 2.5e-06,
@ -41207,7 +41244,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"vertex_ai/claude-fable-5@default": {
"deprecation_date": "2027-06-08",
@ -41241,7 +41279,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"vertex_ai/claude-opus-5": {
"deprecation_date": "2027-01-24",
@ -50152,7 +50191,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
@ -50169,7 +50209,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-4-sonnet": {
"max_tokens": 16384,
@ -50184,7 +50225,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-4-opus": {
"max_tokens": 16384,
@ -50200,7 +50242,8 @@
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-haiku-4-5": {
"max_tokens": 16384,
@ -50215,7 +50258,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 4096
},
"snowflake/claude-3-7-sonnet": {
"max_tokens": 16384,

2
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -23538,7 +23538,7 @@
"paths": {
"/prompts": {
"post": {
"description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer <your_api_key>\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```",
"description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer <your_api_key>\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_data\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```",
"operationId": "create_prompt_prompts_post",
"requestBody": {
"content": {

34
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -14,6 +14,7 @@ import json
import math
import traceback
from collections.abc import Mapping, Sequence
from collections.abc import Set as AbstractSet
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protocol, TypeVar, cast
@ -494,8 +495,13 @@ class TeamMemberBudgetHandler:
team_member_rpm_limit: int | None = None,
team_member_tpm_limit: int | None = None,
team_member_budget_duration: str | None = None,
explicitly_set_fields: AbstractSet[str] = frozenset(),
) -> dict:
"""Create team member budget table with provided limits"""
"""Create team member budget table with provided limits.
The team's own reset period is only inherited when the caller left the
member duration out, so an explicit null means "never resets".
"""
from litellm.proxy._types import BudgetNewRequest
from litellm.proxy.management_endpoints.budget_management_endpoints import (
new_budget,
@ -509,7 +515,11 @@ class TeamMemberBudgetHandler:
# Create budget request with all provided limits
budget_request: Final = BudgetNewRequest(
budget_id=budget_id,
budget_duration=data.budget_duration or team_member_budget_duration,
budget_duration=(
team_member_budget_duration
if "team_member_budget_duration" in explicitly_set_fields
else data.budget_duration or team_member_budget_duration
),
)
if team_member_budget is not None:
@ -545,8 +555,13 @@ class TeamMemberBudgetHandler:
team_member_rpm_limit: int | None = None,
team_member_tpm_limit: int | None = None,
team_member_budget_duration: str | None = None,
explicitly_set_fields: AbstractSet[str] = frozenset(),
) -> dict:
"""Upsert team member budget table with provided limits"""
"""Upsert team member budget table with provided limits.
A field the caller explicitly sent as null is written as null, so a
team can keep a member budget while dropping its reset period.
"""
from litellm.proxy._types import BudgetNewRequest
from litellm.proxy.management_endpoints.budget_management_endpoints import (
update_budget,
@ -560,14 +575,16 @@ class TeamMemberBudgetHandler:
# Budget exists - create update request with only provided values
budget_request: Final = BudgetNewRequest(budget_id=team_member_budget_id)
if team_member_budget is not None:
if team_member_budget is not None or "team_member_budget" in explicitly_set_fields:
budget_request.max_budget = team_member_budget
if team_member_rpm_limit is not None:
if team_member_rpm_limit is not None or "team_member_rpm_limit" in explicitly_set_fields:
budget_request.rpm_limit = team_member_rpm_limit
if team_member_tpm_limit is not None:
if team_member_tpm_limit is not None or "team_member_tpm_limit" in explicitly_set_fields:
budget_request.tpm_limit = team_member_tpm_limit
if team_member_budget_duration is not None:
if team_member_budget_duration is not None or "team_member_budget_duration" in explicitly_set_fields:
budget_request.budget_duration = team_member_budget_duration
if team_member_budget_duration is None:
budget_request.budget_reset_at = None
budget_row: Final = await _as_budget_write(update_budget)(
budget_obj=budget_request,
@ -593,6 +610,7 @@ class TeamMemberBudgetHandler:
team_member_rpm_limit=team_member_rpm_limit,
team_member_tpm_limit=team_member_tpm_limit,
team_member_budget_duration=team_member_budget_duration,
explicitly_set_fields=explicitly_set_fields,
)
# Remove team member fields from updated_kv
@ -1479,6 +1497,7 @@ async def new_team(
team_member_rpm_limit=data.team_member_rpm_limit,
team_member_tpm_limit=data.team_member_tpm_limit,
team_member_budget_duration=data.team_member_budget_duration,
explicitly_set_fields=data.model_fields_set,
)
## ADD TO TEAM TABLE
@ -2184,6 +2203,7 @@ async def update_team(
team_member_rpm_limit=data.team_member_rpm_limit,
team_member_tpm_limit=data.team_member_tpm_limit,
team_member_budget_duration=data.team_member_budget_duration,
explicitly_set_fields=_team_member_fields_in_request,
)
# Backfill team_memberships for members who joined before the
# budget was configured — they won't have a membership row yet.

31
litellm/proxy/prompts/prompt_endpoints.py
View file

@ -323,6 +323,7 @@ def create_versioned_prompt_spec(db_prompt: _PromptRow) -> PromptSpec:
prompt_info=prompt_info,
created_at=row.created_at,
updated_at=row.updated_at,
version=row.version,
environment=row.environment,
created_by=row.created_by,
)
@ -334,6 +335,21 @@ class Prompt(BaseModel):
prompt_info: PromptInfo | None = None
AMBIGUOUS_PROMPT_DATA_ERROR: Final = (
"litellm_params.prompt_id cannot be combined with prompt_data keyed by template name. "
'Send a flat template, prompt_data={"content": "...", "metadata": {...}}, together with litellm_params.prompt_id, '
'or send prompt_data={"<template_id>": {"content": "...", "metadata": {...}}} without litellm_params.prompt_id.'
)
def is_ambiguous_keyed_prompt_data(litellm_params: PromptLiteLLMParams) -> bool:
extra_fields: Final = litellm_params.model_extra or {}
prompt_data: Final = extra_fields.get("prompt_data")
if not litellm_params.prompt_id or not isinstance(prompt_data, dict):
return False
return bool(prompt_data) and "content" not in prompt_data
class PatchPromptRequest(BaseModel):
litellm_params: PromptLiteLLMParams | None = None
prompt_info: PromptInfo | None = None
@ -737,11 +753,9 @@ async def create_prompt(
-d '{
"prompt_id": "my_prompt",
"litellm_params": {
"prompt_id": "json_prompt",
"prompt_id": "my_prompt",
"prompt_integration": "dotprompt",
### EITHER prompt_directory OR prompt_data MUST BE PROVIDED
"prompt_directory": "/path/to/dotprompt/folder",
"prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}}
"prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}
},
"prompt_info": {
"prompt_type": "config"
@ -763,6 +777,9 @@ async def create_prompt(
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if is_ambiguous_keyed_prompt_data(request.litellm_params):
raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR)
try:
# Extract environment from request
environment: Final = (
@ -857,6 +874,9 @@ async def update_prompt(
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if is_ambiguous_keyed_prompt_data(request.litellm_params):
raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR)
try:
# Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success")
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
@ -1086,6 +1106,9 @@ async def patch_prompt(
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if request.litellm_params is not None and is_ambiguous_keyed_prompt_data(request.litellm_params):
raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR)
try:
# Resolve the target row: find the latest version in the given environment
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)

3
litellm/proxy/prompts/prompt_registry.py
View file

@ -147,6 +147,9 @@ class InMemoryPromptRegistry:
prompt_info=prompt.prompt_info or PromptInfo(prompt_type="config"),
created_at=prompt.created_at,
updated_at=prompt.updated_at,
version=prompt.version,
environment=prompt.environment,
created_by=prompt.created_by,
)
# store references to the prompt in memory

9
litellm/proxy/proxy_server.py
View file

@ -6268,7 +6268,14 @@ class ProxyConfig:
):
from litellm.utils import _update_dictionary
combined_router_settings = _update_dictionary(config_router_settings, db_router_settings.param_value)
db_overlay_deferring_empty_lists_to_config: Final = {
k: v
for k, v in db_router_settings.param_value.items()
if not (k in config_router_settings and isinstance(v, list) and len(v) == 0)
}
combined_router_settings = _update_dictionary(
config_router_settings, db_overlay_deferring_empty_lists_to_config
)
elif config_router_settings is not None and isinstance(config_router_settings, dict):
combined_router_settings = config_router_settings
elif db_router_settings is not None and isinstance(db_router_settings.param_value, dict):

140
model_prices_and_context_window.json
View file

@ -1428,7 +1428,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"global.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
@ -1465,7 +1465,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"us.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
@ -1502,7 +1502,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"eu.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
@ -1539,7 +1539,7 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 512
},
"anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2933,7 +2933,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-5": {
"deprecation_date": "2026-10-19",
@ -2956,7 +2957,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-6": {
"deprecation_date": "2027-02-02",
@ -2987,7 +2989,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 4096
},
"azure_ai/claude-opus-4-7": {
"deprecation_date": "2027-04-06",
@ -3018,7 +3021,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 2048
},
"azure_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
@ -3050,7 +3054,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"azure_ai/claude-opus-5": {
"supports_mid_conversation_system": true,
@ -3113,7 +3118,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-opus-4-1": {
"deprecation_date": "2026-08-05",
@ -3135,7 +3141,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-4-5": {
"deprecation_date": "2026-10-19",
@ -3157,7 +3164,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
@ -3188,7 +3196,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 1024
},
"azure_ai/claude-sonnet-4-6": {
"deprecation_date": "2027-02-10",
@ -3214,7 +3223,8 @@
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 1024
},
"azure/computer-use-preview": {
"input_cost_per_token": 3e-06,
@ -14724,7 +14734,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4": {
"cache_creation_input_token_cost": 1.874999e-05,
@ -14746,7 +14757,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-opus-4-1": {
"cache_creation_input_token_cost": 1.874999e-05,
@ -14768,7 +14780,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-opus-4-5": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14791,7 +14804,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14814,7 +14828,8 @@
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 4096
},
"databricks/databricks-claude-opus-4-7": {
"cache_creation_input_token_cost": 6.25002e-06,
@ -14916,7 +14931,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-4-1": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -14960,7 +14976,8 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -14983,7 +15000,8 @@
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"prompt_cache_min_tokens": 1024
},
"databricks/databricks-claude-sonnet-5": {
"cache_creation_input_token_cost": 3.74997e-06,
@ -33899,7 +33917,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-opus-4.1": {
"input_cost_per_image": 0.0048,
@ -33919,7 +33938,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-sonnet-4": {
"input_cost_per_image": 0.0048,
@ -33942,7 +33962,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-sonnet-4.6": {
"supports_adaptive_thinking": true,
@ -33968,7 +33989,8 @@
"supports_reasoning": true,
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
@ -33987,7 +34009,8 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
@ -34008,7 +34031,8 @@
"supports_reasoning": true,
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-sonnet-4.5": {
"input_cost_per_image": 0.0048,
@ -34031,7 +34055,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"openrouter/anthropic/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
@ -34049,7 +34074,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"openrouter/anthropic/claude-opus-4.7": {
"supports_adaptive_thinking": true,
@ -34072,7 +34098,8 @@
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"prompt_cache_min_tokens": 2048
},
"openrouter/anthropic/claude-opus-5": {
"prompt_cache_min_tokens": 512,
@ -36539,7 +36566,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 4096
},
"replicate/ibm-granite/granite-3.3-8b-instruct": {
"input_cost_per_token": 3e-08,
@ -36621,7 +36649,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 1024
},
"replicate/deepseek-ai/deepseek-v3": {
"input_cost_per_token": 1.45e-06,
@ -36696,7 +36725,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true
"supports_prompt_caching": true,
"prompt_cache_min_tokens": 1024
},
"replicate/openai/gpt-4.1": {
"input_cost_per_token": 2e-06,
@ -39725,7 +39755,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-opus-4": {
"cache_creation_input_token_cost": 1.875e-05,
@ -39744,7 +39775,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-opus-4.1": {
"cache_creation_input_token_cost": 1.875e-05,
@ -39763,7 +39795,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
@ -39783,7 +39816,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
@ -39805,7 +39839,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true
"supports_output_config": true,
"prompt_cache_min_tokens": 4096
},
"vercel_ai_gateway/anthropic/claude-sonnet-4": {
"cache_creation_input_token_cost": 3.75e-06,
@ -39824,7 +39859,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/anthropic/claude-sonnet-4.5": {
"cache_creation_input_token_cost": 3.75e-06,
@ -39842,7 +39878,8 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"prompt_cache_min_tokens": 1024
},
"vercel_ai_gateway/cohere/command-a": {
"input_cost_per_token": 2.5e-06,
@ -41207,7 +41244,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"vertex_ai/claude-fable-5@default": {
"deprecation_date": "2027-06-08",
@ -41241,7 +41279,8 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
"supports_max_reasoning_effort": true,
"prompt_cache_min_tokens": 512
},
"vertex_ai/claude-opus-5": {
"deprecation_date": "2027-01-24",
@ -50152,7 +50191,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
@ -50169,7 +50209,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-4-sonnet": {
"max_tokens": 16384,
@ -50184,7 +50225,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-4-opus": {
"max_tokens": 16384,
@ -50200,7 +50242,8 @@
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 1024
},
"snowflake/claude-haiku-4-5": {
"max_tokens": 16384,
@ -50215,7 +50258,8 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_response_schema": true
"supports_response_schema": true,
"prompt_cache_min_tokens": 4096
},
"snowflake/claude-3-7-sonnet": {
"max_tokens": 16384,

80
tests/test_litellm/integrations/dotprompt/test_prompt_manager.py
View file

@ -577,3 +577,83 @@ async def test_dotprompt_with_prompt_version():
)
assert "Version 2:" in v2_rendered
assert "Test v2" in v2_rendered
def test_keyed_prompt_data_with_prompt_id_keeps_real_content():
prompt_data = {
"json_prompt": {
"content": "You are a pirate. Begin every reply with AHOY.",
"metadata": {"model": "gpt-4o-mini"},
}
}
manager = PromptManager(prompt_data=prompt_data, prompt_id="agent-prompt")
template = manager.get_prompt("json_prompt")
assert template is not None
assert template.content == "You are a pirate. Begin every reply with AHOY."
assert template.model == "gpt-4o-mini"
assert "agent-prompt" not in manager.prompts
def test_flat_prompt_data_with_prompt_id_registers_under_prompt_id():
manager = PromptManager(
prompt_data={"content": "Hello {{name}}", "metadata": {"model": "gpt-4o-mini"}},
prompt_id="flat-prompt",
)
template = manager.get_prompt("flat-prompt")
assert template is not None
assert template.content == "Hello {{name}}"
assert manager.render("flat-prompt", {"name": "world"}) == "Hello world"
def test_get_prompt_falls_back_to_base_id_for_versioned_id():
manager = PromptManager(
prompt_data={"content": "Hi", "metadata": {}},
prompt_id="my-prompt",
)
assert manager.get_prompt("my-prompt.v1") is not None
assert manager.get_prompt("my-prompt.v12") is not None
assert manager.get_prompt("my-prompt.vx") is None
assert manager.get_prompt("other-prompt.v1") is None
def test_should_run_prompt_management_accepts_versioned_id():
from litellm.integrations.dotprompt import DotpromptManager
dotprompt_manager = DotpromptManager(
prompt_data={"content": "Hi", "metadata": {}},
prompt_id="versioned-prompt",
)
assert dotprompt_manager.should_run_prompt_management("versioned-prompt", None, {}) is True
assert dotprompt_manager.should_run_prompt_management("versioned-prompt.v1", None, {}) is True
assert dotprompt_manager.should_run_prompt_management("missing-prompt", None, {}) is False
def test_prompt_initializer_registers_flat_db_prompt_under_base_id():
from litellm.integrations.dotprompt import DotpromptManager, prompt_initializer
from litellm.types.prompts.init_prompts import (
PromptInfo,
PromptLiteLLMParams,
PromptSpec,
)
litellm_params = PromptLiteLLMParams(
prompt_integration="dotprompt",
prompt_data={"content": "AHOY {{name}}", "metadata": {"model": "gpt-4o-mini"}},
)
prompt_spec = PromptSpec(
prompt_id="agent-prompt.v1",
litellm_params=litellm_params,
prompt_info=PromptInfo(prompt_type="db"),
)
dotprompt_manager = prompt_initializer(litellm_params, prompt_spec)
assert isinstance(dotprompt_manager, DotpromptManager)
template = dotprompt_manager.prompt_manager.get_prompt("agent-prompt")
assert template is not None
assert template.content == "AHOY {{name}}"

173
tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
View file

@ -2757,3 +2757,176 @@ def test_video_generation_with_input_reference_keeps_file_multipart():
"seconds": "4",
}
assert result.status == "queued"
AZURE_AI_BASE = "https://myfoundry.services.ai.azure.com"
AZURE_AI_CHAT_COMPLETIONS_URL = f"{AZURE_AI_BASE}/models/chat/completions"
def _a_tool_with_an_unsupported_field() -> dict:
return {
"type": "function",
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
"strict": True,
}
A_COMPLETION = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "grok-3",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
TOOL_LEVEL_REJECTION = "Extra inputs are not permitted: tools[0].strict"
UNRELATED_REJECTION = "Extra inputs are not permitted: temperature"
A_REJECTION_THE_PROVIDER_CANNOT_FIX = "The model is not available in this region"
class _RecordedAzureAI:
def __init__(self, responses: list[httpx.Response]) -> None:
self._responses = responses
self.bodies: list[dict] = []
def __call__(self, request: httpx.Request) -> httpx.Response:
self.bodies.append(json.loads(request.content))
return self._responses[min(len(self.bodies) - 1, len(self._responses) - 1)]
@pytest.fixture
def httpx_transport(monkeypatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
def _rejection(message: str) -> httpx.Response:
return httpx.Response(422, json={"error": {"message": message}})
def _call_azure_ai(recorder: _RecordedAzureAI, **overrides):
import respx
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
return litellm.completion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
**overrides,
)
def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried():
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
response = _call_azure_ai(recorder)
assert len(recorder.bodies) == 2
assert recorder.bodies[0]["tools"][0]["strict"] is True
assert "strict" not in recorder.bodies[1]["tools"][0]
assert response.choices[0].message.content == "sent"
def test_the_retry_changes_only_the_field_the_provider_named():
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
_call_azure_ai(recorder)
first, second = recorder.bodies
assert second["messages"] == first["messages"]
assert second["model"] == first["model"]
assert second["tools"][0]["function"] == first["tools"][0]["function"]
def test_a_provider_that_keeps_rejecting_is_not_retried_forever():
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
with pytest.raises(litellm.BadRequestError) as raised:
_call_azure_ai(recorder)
assert len(recorder.bodies) == 2
assert raised.value.status_code == 422
def test_a_rejection_the_provider_cannot_fix_is_not_retried_at_all():
recorder = _RecordedAzureAI([_rejection(A_REJECTION_THE_PROVIDER_CANNOT_FIX)])
with pytest.raises(litellm.BadRequestError):
_call_azure_ai(recorder)
assert len(recorder.bodies) == 1
def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was_asked_for():
recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION)])
with pytest.raises(litellm.BadRequestError):
_call_azure_ai(recorder)
assert len(recorder.bodies) == 1
def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for():
recorder = _RecordedAzureAI(
[_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
response = _call_azure_ai(recorder, drop_params=True)
assert len(recorder.bodies) == 2
assert response.choices[0].message.content == "sent"
@pytest.mark.asyncio
async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_async_path(
httpx_transport,
):
import respx
recorder = _RecordedAzureAI(
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
)
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
response = await litellm.acompletion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
)
assert len(recorder.bodies) == 2
assert recorder.bodies[0]["tools"][0]["strict"] is True
assert "strict" not in recorder.bodies[1]["tools"][0]
assert response.choices[0].message.content == "sent"
@pytest.mark.asyncio
async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_async_path(
httpx_transport,
):
import respx
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
with respx.mock(assert_all_called=True) as router:
router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder)
with pytest.raises(litellm.BadRequestError):
await litellm.acompletion(
model="azure_ai/grok-3",
messages=[{"role": "user", "content": "hi"}],
tools=[_a_tool_with_an_unsupported_field()],
api_base=AZURE_AI_BASE,
api_key="fake-key",
)
assert len(recorder.bodies) == 2

134
tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
View file

@ -2366,6 +2366,7 @@ async def test_update_team_team_member_budget_not_passed_to_db(
team_member_rpm_limit=None,
team_member_tpm_limit=None,
team_member_budget_duration=None,
explicitly_set_fields=frozenset(),
):
# Remove team_member_budget from updated_kv as the real function does
result_kv = updated_kv.copy()
@ -2738,6 +2739,138 @@ async def test_upsert_team_member_budget_table_no_existing_budget():
assert "team_member_budget_duration" not in result
@pytest.mark.asyncio
async def test_upsert_team_member_budget_table_clears_duration_kept_budget(mock_db_client):
"""
A request that keeps team_member_budget but explicitly nulls
team_member_budget_duration must clear the reset period and its reset time.
"""
from litellm.proxy.management_endpoints.team_endpoints import (
TeamMemberBudgetHandler,
)
mock_user_api_key_dict = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id"
)
team_table = MagicMock(spec=LiteLLM_TeamTable)
team_table.metadata = {"team_member_budget_id": "existing_budget_123"}
mock_db_client.db.litellm_budgettable.update = AsyncMock(
side_effect=lambda where, data: SimpleNamespace(**data)
)
result = await TeamMemberBudgetHandler.upsert_team_member_budget_table(
team_table=team_table,
user_api_key_dict=mock_user_api_key_dict,
updated_kv={
"team_id": "test_team_id",
"team_member_budget": 100.0,
"team_member_budget_duration": None,
},
team_member_budget=100.0,
team_member_budget_duration=None,
explicitly_set_fields={
"team_member_budget",
"team_member_budget_duration",
},
)
written = mock_db_client.db.litellm_budgettable.update.call_args.kwargs["data"]
assert written["max_budget"] == 100.0
assert written["budget_duration"] is None
assert written["budget_reset_at"] is None
assert "rpm_limit" not in written
assert "tpm_limit" not in written
assert result["metadata"]["team_member_budget_id"] == "existing_budget_123"
assert "team_member_budget" not in result
assert "team_member_budget_duration" not in result
@pytest.mark.asyncio
async def test_create_team_member_budget_table_explicit_null_duration_does_not_inherit_team_duration(
mock_db_client,
):
"""
A first-time member budget with an explicitly null duration must never
reset, even when the team itself has a reset period.
"""
from litellm.proxy.management_endpoints.team_endpoints import (
TeamMemberBudgetHandler,
)
mock_user_api_key_dict = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id"
)
team_table = MagicMock(spec=LiteLLM_TeamTable)
team_table.metadata = {}
team_table.team_alias = "Test Team"
team_table.budget_duration = "30d"
mock_db_client.db.litellm_budgettable.create = AsyncMock(
side_effect=lambda data: SimpleNamespace(**data)
)
result = await TeamMemberBudgetHandler.create_team_member_budget_table(
data=team_table,
new_team_data_json={"team_id": "test_team_id"},
user_api_key_dict=mock_user_api_key_dict,
team_member_budget=100.0,
team_member_budget_duration=None,
explicitly_set_fields={
"team_member_budget",
"team_member_budget_duration",
},
)
written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"]
assert written["max_budget"] == 100.0
assert "budget_duration" not in written
assert "budget_reset_at" not in written
assert result["metadata"]["team_member_budget_id"] == written["budget_id"]
assert "team_member_budget" not in result
@pytest.mark.asyncio
async def test_create_team_member_budget_table_inherits_team_duration_when_duration_omitted(
mock_db_client,
):
"""
Omitting team_member_budget_duration keeps the existing inheritance of the
team's own reset period.
"""
from litellm.proxy.management_endpoints.team_endpoints import (
TeamMemberBudgetHandler,
)
mock_user_api_key_dict = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id"
)
team_table = MagicMock(spec=LiteLLM_TeamTable)
team_table.metadata = {}
team_table.team_alias = "Test Team"
team_table.budget_duration = "30d"
mock_db_client.db.litellm_budgettable.create = AsyncMock(
side_effect=lambda data: SimpleNamespace(**data)
)
result = await TeamMemberBudgetHandler.create_team_member_budget_table(
data=team_table,
new_team_data_json={"team_id": "test_team_id"},
user_api_key_dict=mock_user_api_key_dict,
team_member_budget=100.0,
explicitly_set_fields={"team_member_budget"},
)
written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"]
assert written["budget_duration"] == "30d"
assert written["budget_reset_at"] is not None
assert result["metadata"]["team_member_budget_id"] == written["budget_id"]
@pytest.mark.asyncio
async def test_update_team_with_team_member_budget_duration(
disable_audit_logging_for_mocked_team,
@ -2799,6 +2932,7 @@ async def test_update_team_with_team_member_budget_duration(
team_member_rpm_limit=None,
team_member_tpm_limit=None,
team_member_budget_duration=None,
explicitly_set_fields=frozenset(),
):
result_kv = updated_kv.copy()
result_kv.pop("team_member_budget", None)

239
tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py
View file

@ -1,3 +1,4 @@
import json
import pytest
from unittest.mock import MagicMock, AsyncMock, patch
from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
@ -246,3 +247,241 @@ async def test_patch_prompt_row_deleted_mid_update_returns_404():
exc_info.value.detail
== "Prompt with ID test_prompt not found in environment development"
)
def test_is_ambiguous_keyed_prompt_data_shapes():
from litellm.proxy.prompts.prompt_endpoints import is_ambiguous_keyed_prompt_data
keyed_with_id = PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
)
flat_with_id = PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"content": "AHOY", "metadata": {}},
)
keyed_without_id = PromptLiteLLMParams(
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
)
no_prompt_data = PromptLiteLLMParams(
prompt_id="agent-prompt", prompt_integration="dotprompt"
)
empty_prompt_data = PromptLiteLLMParams(
prompt_id="agent-prompt", prompt_integration="dotprompt", prompt_data={}
)
assert is_ambiguous_keyed_prompt_data(keyed_with_id) is True
assert is_ambiguous_keyed_prompt_data(flat_with_id) is False
assert is_ambiguous_keyed_prompt_data(keyed_without_id) is False
assert is_ambiguous_keyed_prompt_data(no_prompt_data) is False
assert is_ambiguous_keyed_prompt_data(empty_prompt_data) is False
@pytest.mark.asyncio
async def test_create_prompt_rejects_keyed_prompt_data_with_prompt_id():
from fastapi import HTTPException
from litellm.proxy.prompts.prompt_endpoints import (
AMBIGUOUS_PROMPT_DATA_ERROR,
Prompt,
create_prompt,
)
mock_user_auth = UserAPIKeyAuth(
api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN
)
request = Prompt(
prompt_id="agent-prompt",
litellm_params=PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
),
)
with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point
with pytest.raises(HTTPException) as exc_info:
await create_prompt(request=request, user_api_key_dict=mock_user_auth)
assert exc_info.value.status_code == 400
assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR
@pytest.mark.asyncio
async def test_patch_prompt_rejects_keyed_prompt_data_with_prompt_id():
from fastapi import HTTPException
from litellm.proxy.prompts.prompt_endpoints import (
AMBIGUOUS_PROMPT_DATA_ERROR,
PatchPromptRequest,
patch_prompt,
)
mock_user_auth = UserAPIKeyAuth(
api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN
)
request = PatchPromptRequest(
litellm_params=PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
),
)
with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point
with pytest.raises(HTTPException) as exc_info:
await patch_prompt(
prompt_id="agent-prompt",
request=request,
user_api_key_dict=mock_user_auth,
)
assert exc_info.value.status_code == 400
assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR
@pytest.mark.asyncio
async def test_patch_prompt_info_only_keeps_legacy_keyed_row_patchable():
from litellm.proxy.prompts.prompt_endpoints import PatchPromptRequest, patch_prompt
mock_user_auth = UserAPIKeyAuth(
api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN
)
legacy_params = PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
)
target_row = MagicMock()
target_row.id = "row-1"
target_row.version = 1
updated_row = MagicMock()
updated_row.model_dump.return_value = {
"prompt_id": "agent-prompt",
"version": 1,
"environment": "production",
"created_by": None,
"litellm_params": legacy_params.model_dump_json(),
"prompt_info": PromptInfo(prompt_type="db", environment="production").model_dump_json(),
}
mock_prisma_client = MagicMock()
mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock(
return_value=[target_row]
)
mock_prisma_client.db.litellm_prompttable.update = AsyncMock(return_value=updated_row)
existing_prompt = PromptSpec(
prompt_id="agent-prompt.v1",
litellm_params=legacy_params,
prompt_info=PromptInfo(prompt_type="db"),
)
with (
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), # test-quality-ok: proxy_server module global is the endpoint's only injection point
patch( # test-quality-ok: keeps the registry reload from touching global callback state
"litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY"
) as mock_registry,
):
mock_registry.get_prompt_by_id.return_value = existing_prompt
await patch_prompt(
prompt_id="agent-prompt",
request=PatchPromptRequest(prompt_info=PromptInfo(prompt_type="db", environment="production")),
user_api_key_dict=mock_user_auth,
)
update_kwargs = mock_prisma_client.db.litellm_prompttable.update.await_args.kwargs
assert update_kwargs["where"] == {"id": "row-1"}
assert json.loads(update_kwargs["data"]["prompt_info"])["environment"] == "production"
assert json.loads(update_kwargs["data"]["litellm_params"])["prompt_data"] == {
"json_prompt": {"content": "AHOY", "metadata": {}}
}
@pytest.mark.asyncio
async def test_update_prompt_rejects_keyed_prompt_data_with_prompt_id():
from fastapi import HTTPException
from litellm.proxy.prompts.prompt_endpoints import (
AMBIGUOUS_PROMPT_DATA_ERROR,
Prompt,
update_prompt,
)
mock_user_auth = UserAPIKeyAuth(
api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN
)
request = Prompt(
prompt_id="agent-prompt",
litellm_params=PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}},
),
)
with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point
with pytest.raises(HTTPException) as exc_info:
await update_prompt(
prompt_id="agent-prompt",
request=request,
user_api_key_dict=mock_user_auth,
)
assert exc_info.value.status_code == 400
assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR
def test_create_versioned_prompt_spec_populates_version():
from litellm.proxy.prompts.prompt_endpoints import create_versioned_prompt_spec
db_prompt = MagicMock()
db_prompt.model_dump.return_value = {
"prompt_id": "agent-prompt",
"version": 3,
"environment": "development",
"created_by": "user-1",
"litellm_params": {
"prompt_id": "agent-prompt",
"prompt_integration": "dotprompt",
},
"prompt_info": {"prompt_type": "db"},
"created_at": None,
"updated_at": None,
}
prompt_spec = create_versioned_prompt_spec(db_prompt=db_prompt)
assert prompt_spec.prompt_id == "agent-prompt.v3"
assert prompt_spec.version == 3
def test_initialize_prompt_keeps_version_and_created_by():
import litellm
from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry
registry = InMemoryPromptRegistry()
prompt_spec = PromptSpec(
prompt_id="agent-prompt.v3",
litellm_params=PromptLiteLLMParams(
prompt_id="agent-prompt",
prompt_integration="dotprompt",
prompt_data={"content": "AHOY", "metadata": {}},
),
prompt_info=PromptInfo(prompt_type="db"),
version=3,
environment="development",
created_by="user-1",
)
with patch.object(litellm.logging_callback_manager, "add_litellm_callback"): # test-quality-ok: keeps initialize_prompt from registering a global callback that would leak across tests
initialized_prompt = registry.initialize_prompt(prompt=prompt_spec)
assert initialized_prompt is not None
assert initialized_prompt.version == 3
assert initialized_prompt.created_by == "user-1"
assert initialized_prompt.environment == "development"

88
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -4741,6 +4741,90 @@ async def test_add_router_settings_from_db_config_merge_logic():
assert combined_settings["nested_config"] == expected_nested
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks():
"""
Regression test for DB router_settings rows carrying explicit empty lists
(e.g. {"fallbacks": []} written by the dashboard's delete-last-fallback flow):
empty lists are "no value" and must not clobber config.yaml fallbacks,
matching _deep_merge_dicts semantics. Non-empty DB lists still win.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
mock_router = MagicMock()
mock_router.update_settings = MagicMock()
config_data = {
"router_settings": {
"fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}],
"context_window_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}],
"content_policy_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}],
}
}
mock_db_config = MagicMock()
mock_db_config.param_value = {
"fallbacks": [],
"context_window_fallbacks": [],
"content_policy_fallbacks": [{"gpt-oss-120b": ["other-model"]}],
"model_group_alias": {},
"num_retries": 3,
}
mock_prisma_client = MagicMock()
mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config)
await proxy_config._add_router_settings_from_db_config(
config_data=config_data,
llm_router=mock_router,
prisma_client=mock_prisma_client,
)
combined_settings = mock_router.update_settings.call_args.kwargs
assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["other-model"]}]
assert combined_settings["num_retries"] == 3
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_empty_db_list_still_clears_unconfigured_key():
"""
An empty DB list only yields to config.yaml where the yaml configures that key.
When the yaml router_settings has no fallbacks, a DB {"fallbacks": []} (the
dashboard's delete-last-fallback write) must still reach the router so the
running pods drop the deleted fallback without a restart.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
mock_router = MagicMock()
mock_router.update_settings = MagicMock()
config_data = {"router_settings": {"num_retries": 1}}
mock_db_config = MagicMock()
mock_db_config.param_value = {"fallbacks": [], "model_group_alias": {}}
mock_prisma_client = MagicMock()
mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config)
await proxy_config._add_router_settings_from_db_config(
config_data=config_data,
llm_router=mock_router,
prisma_client=mock_prisma_client,
)
combined_settings = mock_router.update_settings.call_args.kwargs
assert combined_settings["fallbacks"] == []
assert combined_settings["num_retries"] == 1
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_edge_cases():
"""
@ -11304,9 +11388,7 @@ class TestRouterModelNameOnStreamingChunks:
with patch.object(ProxyLogging, "_fire_deferred_stream_logging"):
return [
data
async for data in async_data_generator(
mock_response, MagicMock(spec=UserAPIKeyAuth), request_data
)
async for data in async_data_generator(mock_response, MagicMock(spec=UserAPIKeyAuth), request_data)
]
@staticmethod

161
tests/test_litellm/test_cost_calculator.py
View file

@ -3909,6 +3909,167 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_
assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9)
def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map):
"""A router-facing model_name alias containing "/" whose leading segment is NOT a
registered provider must not be double-prefixed into a non-existent cost key.
Regression test for #38069: alias "vertex/claude-opus-5" (real deployment
"vertex_ai/claude-opus-5") was re-prefixed into "vertex_ai/vertex/claude-opus-5",
silently pricing every streamed request at $0.
"""
from litellm.cost_calculator import _select_model_name_for_cost_calc
response = litellm.ModelResponse(
id="x",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
model="vertex/claude-opus-5",
)
response._hidden_params = {}
selected = _select_model_name_for_cost_calc(
model=None,
completion_response=response,
custom_llm_provider="vertex_ai",
)
assert selected == "vertex_ai/claude-opus-5"
def test_select_model_name_strips_duplicated_region_segment(_local_model_cost_map):
"""A "region/model" alias whose leading segment repeats the request's region must
resolve to the region-priced cost key instead of keeping the region segment twice."""
from litellm.cost_calculator import _select_model_name_for_cost_calc
response = litellm.ModelResponse(
id="x",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
model="us-east-1/anthropic.claude-v2:1",
)
response._hidden_params = {"region_name": "us-east-1"}
selected = _select_model_name_for_cost_calc(
model=None,
completion_response=response,
custom_llm_provider="bedrock",
)
assert selected == "bedrock/us-east-1/anthropic.claude-v2:1"
def test_completion_cost_nonzero_for_slash_alias_model_name(_local_model_cost_map):
"""End-to-end cost through a "/"-containing alias must price above zero (#38069)."""
response = litellm.ModelResponse(
id="x",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
model="vertex/claude-opus-5",
)
response._hidden_params = {"custom_llm_provider": "vertex_ai"}
response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50)
cost = litellm.completion_cost(
completion_response=response,
custom_llm_provider="vertex_ai",
)
assert cost == pytest.approx(100 * 5e-6 + 50 * 2.5e-5, rel=1e-9)
def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map):
"""An alias that resolves to no known cost key keeps the legacy double-prefixed name."""
from litellm.cost_calculator import _select_model_name_for_cost_calc
response = litellm.ModelResponse(
id="x",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
model="team/nonsense-model",
)
response._hidden_params = {}
selected = _select_model_name_for_cost_calc(
model=None,
completion_response=response,
custom_llm_provider="vertex_ai",
)
assert selected == "vertex_ai/team/nonsense-model"
def test_completion_cost_keeps_custom_priced_slash_router_id(_local_model_cost_map):
"""A custom-priced router id containing "/" keeps its custom pricing instead of being
rewritten to the built-in key its suffix happens to match."""
from litellm.cost_calculator import _select_model_name_for_cost_calc
litellm.register_model(
model_cost={
"vertex/claude-opus-5": {
"input_cost_per_token": 7e-6,
"output_cost_per_token": 8e-6,
"litellm_provider": "vertex_ai",
}
}
)
selected = _select_model_name_for_cost_calc(
model="vertex_ai/claude-opus-5",
completion_response=None,
custom_pricing=True,
custom_llm_provider="vertex_ai",
router_model_id="vertex/claude-opus-5",
)
assert selected == "vertex_ai/vertex/claude-opus-5"
response = litellm.ModelResponse(
id="x",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
model="vertex/claude-opus-5",
)
response._hidden_params = {"custom_llm_provider": "vertex_ai"}
response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50)
cost = litellm.completion_cost(
completion_response=response,
custom_llm_provider="vertex_ai",
custom_pricing=True,
router_model_id="vertex/claude-opus-5",
)
assert cost == pytest.approx(100 * 7e-6 + 50 * 8e-6, rel=1e-9)
@pytest.mark.parametrize(
("model", "expected_1hr_rate"),
[("claude-3-haiku-20240307", 5e-07), ("claude-3-opus-20240229", 3e-05)],

103
tests/test_litellm/test_utils.py
View file

@ -4452,14 +4452,101 @@ def test_get_prompt_cache_min_tokens_resolves_per_model(
assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens
def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None:
"""The same model can carry a different minimum per platform, so the threshold must come from
the platform's own cost-map entry rather than being derived from the model family name."""
assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512
assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024
assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens(
model="anthropic.claude-fable-5"
)
def test_get_prompt_cache_min_tokens_uniform_for_fable_5_across_platforms(local_model_cost_map: None) -> None:
"""Anthropic removed the Amazon Bedrock override for Claude Fable 5, so its 512-token minimum
now applies on every platform. The Bedrock entries carried the old 1024 and the re-export
entries carried nothing, so the router judged 512-1023-token prefixes uncacheable and skipped
prompt-cache-affinity routing for prompts the provider demonstrably caches (issue #35011)."""
wrong: Final = {
model: get_prompt_cache_min_tokens(model=model)
for model, info in litellm.model_cost.items()
if "fable-5" in model
and info.get("supports_prompt_caching")
and get_prompt_cache_min_tokens(model=model) != 512
}
assert not wrong, f"every Claude Fable 5 entry must carry prompt_cache_min_tokens 512: {wrong}"
ANTHROPIC_REEXPORT_CACHE_MIN: Final = {
"azure_ai/claude-fable-5": 512,
"azure_ai/claude-haiku-4-5": 4096,
"azure_ai/claude-opus-4-1": 1024,
"azure_ai/claude-opus-4-5": 4096,
"azure_ai/claude-opus-4-6": 4096,
"azure_ai/claude-opus-4-7": 2048,
"azure_ai/claude-opus-4-8": 1024,
"azure_ai/claude-sonnet-4-5": 1024,
"azure_ai/claude-sonnet-4-6": 1024,
"azure_ai/claude-sonnet-5": 1024,
"databricks/databricks-claude-haiku-4-5": 4096,
"databricks/databricks-claude-opus-4": 1024,
"databricks/databricks-claude-opus-4-1": 1024,
"databricks/databricks-claude-opus-4-5": 4096,
"databricks/databricks-claude-opus-4-6": 4096,
"databricks/databricks-claude-sonnet-4": 1024,
"databricks/databricks-claude-sonnet-4-5": 1024,
"databricks/databricks-claude-sonnet-4-6": 1024,
"openrouter/anthropic/claude-haiku-4.5": 4096,
"openrouter/anthropic/claude-opus-4": 1024,
"openrouter/anthropic/claude-opus-4.1": 1024,
"openrouter/anthropic/claude-opus-4.5": 4096,
"openrouter/anthropic/claude-opus-4.6": 4096,
"openrouter/anthropic/claude-opus-4.7": 2048,
"openrouter/anthropic/claude-sonnet-4": 1024,
"openrouter/anthropic/claude-sonnet-4.5": 1024,
"openrouter/anthropic/claude-sonnet-4.6": 1024,
"replicate/anthropic/claude-4-sonnet": 1024,
"replicate/anthropic/claude-4.5-haiku": 4096,
"replicate/anthropic/claude-4.5-sonnet": 1024,
"snowflake/claude-4-opus": 1024,
"snowflake/claude-4-sonnet": 1024,
"snowflake/claude-haiku-4-5": 4096,
"snowflake/claude-sonnet-4-5": 1024,
"snowflake/claude-sonnet-4-6": 1024,
"vercel_ai_gateway/anthropic/claude-haiku-4.5": 4096,
"vercel_ai_gateway/anthropic/claude-opus-4": 1024,
"vercel_ai_gateway/anthropic/claude-opus-4.1": 1024,
"vercel_ai_gateway/anthropic/claude-opus-4.5": 4096,
"vercel_ai_gateway/anthropic/claude-opus-4.6": 4096,
"vercel_ai_gateway/anthropic/claude-sonnet-4": 1024,
"vercel_ai_gateway/anthropic/claude-sonnet-4.5": 1024,
"vertex_ai/claude-fable-5": 512,
"vertex_ai/claude-fable-5@default": 512,
}
def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local_model_cost_map: None) -> None:
"""Regression for issue #35011: these re-export entries carried no prompt_cache_min_tokens, so
they silently inherited the 1024 default. That skipped cache-affinity routing for Fable 5's
512-1023-token prefixes and reported 1024-4095-token prompts as cacheable on the 2048/4096
models. The entry must be explicit so a default change can never re-break them, which is why
this asserts the cost-map value itself and not just the resolver's answer."""
wrong: Final = {
model: (litellm.model_cost[model].get("prompt_cache_min_tokens"), get_prompt_cache_min_tokens(model=model))
for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items()
if litellm.model_cost[model].get("prompt_cache_min_tokens") != expected
or get_prompt_cache_min_tokens(model=model) != expected
}
assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}"
def test_anthropic_reexport_cache_minimums_present_in_root_cost_map() -> None:
"""The root map ships to the CDN independently of the bundled backup, so both must carry the
minimum or proxies reading one of them regress to the 1024 default."""
root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json")
with open(root_map_path) as f:
root_map: Final = json.load(f)
wrong: Final = {
model: root_map[model].get("prompt_cache_min_tokens")
for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items()
if root_map[model].get("prompt_cache_min_tokens") != expected
}
fable_5_wrong: Final = {
model: info.get("prompt_cache_min_tokens")
for model, info in root_map.items()
if "fable-5" in model and info.get("supports_prompt_caching") and info.get("prompt_cache_min_tokens") != 512
}
assert not wrong and not fable_5_wrong, f"root cost map diverges: {wrong | fable_5_wrong}"
GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(

47
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -17,15 +17,14 @@ import {
ReasoningEffort,
TierModelParamsByTier,
pruneTierModelParams,
resolveComplexityDefaultModel,
setTierModelReasoningEffort,
tierOptions,
} from "./complexity_router_tiers";
import TierModelEffortRows from "./TierModelEffortRows";
import EscalationKeywords from "./EscalationKeywords";
import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
import SemanticKeywordMatching from "./SemanticKeywordMatching";
import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
import { type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows";
export type { DimensionWeights, TierBoundaries, TokenThresholds };
@ -37,12 +36,12 @@ export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120;
export const DEFAULT_SESSION_AFFINITY = false;
export const DEFAULT_DEPLOYMENT_AFFINITY = true;
export interface ComplexityTiers {
export type ComplexityTiers = {
SIMPLE: string[];
MEDIUM: string[];
COMPLEX: string[];
REASONING: string[];
}
};
export type ClassificationRubric = "legacy" | "agentic" | "chat" | "business";
@ -224,10 +223,6 @@ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array<keyof Complexit
export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: ComplexityTierLabels | undefined): string =>
tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label;
/** Tiers the plan-mode floor may name: the backend rejects a floor whose tier has no models. */
export const planModeEligibleTiers = (tiers: ComplexityTiers): Array<keyof ComplexityTiers> =>
TIER_KEYS.filter((tier) => (tiers[tier] ?? []).length > 0);
const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
modelInfo,
value,
@ -246,12 +241,12 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
onEscalationKeywordsChange,
showValidationErrors = false,
}) => {
const planModeTiers = planModeEligibleTiers(value.tiers);
const planModeTierOptions = tierOptions(value.tier_labels).filter((option) =>
(planModeTiers as string[]).includes(option.value),
);
const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers);
const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model);
const tierRows = activeTierRows(value);
const planModeTierOptions = tierRows
.filter((row) => row.models.length > 0)
.map((row) => ({ value: row.id, label: effectiveTierLabel(row.id as keyof ComplexityTiers, value.tier_labels) }));
const derivedDefaultModel = resolveComplexityDefaultModel(value);
const defaultModel = resolveComplexityDefaultModel(value, value.default_model);
// An absent list means the proxy does not send the field yet, so every level is offered as before.
// An empty list is the group's own answer that its deployments share no level, and is left empty.
@ -325,12 +320,13 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
<Card>
<CardContent>
{TIER_KEYS.map((tier, index) => {
{tierRows.map((row: TierRow, index) => {
const tier = row.id as keyof ComplexityTiers;
const tierInfo = TIER_DESCRIPTIONS[tier];
const label = effectiveTierLabel(tier, value.tier_labels);
const tierMissing = showValidationErrors && value.tiers[tier].length === 0;
const tierMissing = showValidationErrors && row.models.length === 0;
return (
<div key={tier}>
<div key={row.id}>
{index > 0 && <Separator className="my-4" />}
<div className="mb-4">
<div className="flex items-center gap-2 mb-2">
@ -339,7 +335,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
<Info className="size-4 text-muted-foreground" />
</SimpleTooltip>
<span className="text-xs text-muted-foreground">
Tier {index + 1} of {TIER_KEYS.length} &middot; {tier}
Tier {index + 1} of {tierRows.length} &middot; {row.id}
</span>
</div>
<span className="block mb-2 text-xs text-muted-foreground">Examples: {tierInfo.examples}</span>
@ -364,7 +360,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
</InputGroup>
<MultiSelect
options={modelOptions}
value={value.tiers[tier]}
value={row.models}
onValueChange={(models: string[]) => handleTierChange(tier, models)}
placeholder={`Select model(s) for ${label.toLowerCase()} queries`}
emptyText="No models found"
@ -372,12 +368,12 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
/>
<TierModelEffortRows
tierLabel={label}
models={value.tiers[tier]}
models={row.models}
effortOptionsByModel={effortOptionsByModel}
paramsByModel={value.tier_model_params?.[tier]}
onEffortChange={(model, effort) => handleTierModelEffortChange(tier, model, effort)}
/>
{value.tiers[tier].length > 1 && (
{row.models.length > 1 && (
<span className="text-xs text-muted-foreground">
Multiple models selected — the router randomly picks among them per request (or Thompson-samples
within the pool when adaptive routing is on).
@ -483,9 +479,12 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
<div className="flex items-center gap-2 mb-2">
<Switch
checked={value.plan_mode_min_tier !== undefined}
disabled={planModeTiers.length === 0}
disabled={planModeTierOptions.length === 0}
onCheckedChange={(enabled) =>
onChange({ ...value, plan_mode_min_tier: enabled ? planModeTiers.at(-1) : undefined })
onChange({
...value,
plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined,
})
}
aria-label="Route plan-mode requests to a minimum tier"
/>
@ -494,7 +493,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
<span className="block text-xs mb-3 text-muted-foreground">
Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier.
The classifier still wins when it picks higher, and the override only lasts while plan mode is active.
{planModeTiers.length === 0 && " Add models to a tier to enable this."}
{planModeTierOptions.length === 0 && " Add models to a tier to enable this."}
</span>
{value.plan_mode_min_tier !== undefined && (
<div style={{ maxWidth: 320 }}>

43
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
View file

@ -22,7 +22,6 @@ import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels";
import ComplexityRouterConfig, {
ComplexityRouterConfigValue,
ComplexityTiers,
DEFAULT_ADAPTIVE_WEIGHTS,
DEFAULT_SESSION_AFFINITY,
DEFAULT_DEPLOYMENT_AFFINITY,
@ -40,7 +39,9 @@ import {
getSemanticConfigError,
getTierLabelsError,
} from "./build_complexity_router_config";
import { resolveComplexityDefaultModel } from "./complexity_router_tiers";
import { activeTierName, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows";
import { DEFAULT_TIER_LABELS } from "./complexity_router_tiers";
import type { ComplexityTier } from "./KeywordTierRules";
import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
import AutoRouterConnectionTest from "./auto_router_connection_test";
import AutoRouterRoutingTest from "./AutoRouterRoutingTest";
@ -104,17 +105,10 @@ const presets = getAllPresets();
// A one-line summary of what's configured, shown when the detailed section is collapsed so a
// caller can see the shape of the config without opening it.
const tierConfigSummary = (tiers: ComplexityTiers): string => {
const parts = (
[
["Simple", tiers.SIMPLE],
["Medium", tiers.MEDIUM],
["Complex", tiers.COMPLEX],
["Reasoning", tiers.REASONING],
] as const
)
.filter(([, models]) => models.length > 0)
.map(([label, models]) => `${label}: ${models.join(", ")}`);
const tierConfigSummary = (config: ComplexityRouterConfigValue): string => {
const parts = activeTierRows(config)
.filter((row) => row.models.length > 0)
.map((row) => `${DEFAULT_TIER_LABELS[row.id as ComplexityTier] ?? activeTierName(row)}: ${row.models.join(", ")}`);
return parts.length > 0 ? parts.join(" · ") : "No tiers configured yet";
};
@ -128,9 +122,9 @@ const getSubmitBlockedReason = (
referencedModelsParams: Parameters<typeof getReferencedModelsError>[0],
availability: ModelAvailability,
): string | null =>
getMissingTiersError(config.tiers) ??
getMissingTiersError(activeTierRows(config)) ??
getTierLabelsError(config.tier_labels) ??
getPlanModeTierError(config.plan_mode_min_tier, config.tiers) ??
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
getKeywordTierRulesError(keywordTierRules) ??
getReferencedModelsError(referencedModelsParams, availability);
@ -378,7 +372,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const submitRecommendedRouter = async (name: string) => {
const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams;
const missingTiersError = getMissingTiersError(tiers);
const missingTiersError = getMissingTiersError(activeTierRows(complexityRouterConfig));
if (missingTiersError) {
setShowValidationErrors(true);
toast.fromError(missingTiersError);
@ -423,7 +417,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
return;
}
const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model);
const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model);
const validatedFields = requiresTeamScope
? (["auto_router_name", "team_id"] as const)
: (["auto_router_name"] as const);
@ -463,10 +457,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const handleTestConnection = () => {
const testTargetParams = {
tiers: complexityRouterConfig.tiers,
tiers: activeTierRows(complexityRouterConfig).map(
(row) => [activeTierName(row), row.models] as [string, string[]],
),
semanticMatchingEnabled,
embeddingModel,
defaultModel: resolveComplexityDefaultModel(complexityRouterConfig.tiers, complexityRouterConfig.default_model),
defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model),
};
const targets = buildAutoRouterTestTargets(testTargetParams);
@ -581,7 +577,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
</span>
{!detailsExpanded && (
<span className="text-xs text-muted-foreground line-clamp-2">
{tierConfigSummary(complexityRouterConfig.tiers)}
{tierConfigSummary(complexityRouterConfig)}
</span>
)}
</button>
@ -694,10 +690,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
<AutoRouterRoutingTest
accessToken={accessToken}
config={buildComplexityRouterConfig(complexityRouterConfigParams)}
defaultModel={resolveComplexityDefaultModel(
complexityRouterConfig.tiers,
complexityRouterConfig.default_model,
)}
defaultModel={resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model)}
routerName={watchedName}
teamId={requiresTeamScope ? watchedTeamId : undefined}
/>
@ -707,7 +700,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
<Button variant="outline" onClick={() => setIsRoutingTestVisible(false)}>
Close
</Button>
, ]
</DialogFooter>
</DialogContent>
</Dialog>
@ -744,7 +736,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
>
Close
</Button>
, ]
</DialogFooter>
</DialogContent>
</Dialog>

33
ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts
View file

@ -1,11 +1,18 @@
import { buildAutoRouterTestTargets } from "./build_auto_router_test_targets";
const tiers = {
SIMPLE: ["gpt-4o-mini"],
MEDIUM: ["claude-sonnet-4"],
COMPLEX: ["claude-sonnet-4"],
REASONING: ["o3"],
};
const tierEntries = (
SIMPLE: string[],
MEDIUM: string[] = [],
COMPLEX: string[] = [],
REASONING: string[] = [],
): [string, string[]][] => [
["SIMPLE", SIMPLE],
["MEDIUM", MEDIUM],
["COMPLEX", COMPLEX],
["REASONING", REASONING],
];
const tiers = tierEntries(["gpt-4o-mini"], ["claude-sonnet-4"], ["claude-sonnet-4"], ["o3"]);
describe("buildAutoRouterTestTargets", () => {
it("dedups tiers that share a model group into one chat target carrying both labels", () => {
@ -19,7 +26,7 @@ describe("buildAutoRouterTestTargets", () => {
it("emits a target per model when a tier has more than one, and dedups across tiers", () => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini", "claude-sonnet-4"], MEDIUM: ["claude-sonnet-4"], COMPLEX: [], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini", "claude-sonnet-4"], ["claude-sonnet-4"]),
semanticMatchingEnabled: false,
embeddingModel: undefined,
});
@ -31,7 +38,7 @@ describe("buildAutoRouterTestTargets", () => {
it("drops empty/whitespace tiers", () => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [" "], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini"], [], [" "]),
semanticMatchingEnabled: false,
embeddingModel: undefined,
});
@ -41,7 +48,7 @@ describe("buildAutoRouterTestTargets", () => {
it("returns [] when no tier is configured", () => {
expect(
buildAutoRouterTestTargets({
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
tiers: tierEntries([]),
semanticMatchingEnabled: false,
embeddingModel: undefined,
}),
@ -50,7 +57,7 @@ describe("buildAutoRouterTestTargets", () => {
it("appends an embedding target only when semantic matching is on and a model is set", () => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini"]),
semanticMatchingEnabled: true,
embeddingModel: "voyage-3-5",
});
@ -62,7 +69,7 @@ describe("buildAutoRouterTestTargets", () => {
it("omits the embedding target when semantic matching is on but no model is chosen", () => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini"]),
semanticMatchingEnabled: true,
embeddingModel: undefined,
});
@ -71,7 +78,7 @@ describe("buildAutoRouterTestTargets", () => {
it("omits the embedding target when a model is set but semantic matching is off", () => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini"]),
semanticMatchingEnabled: false,
embeddingModel: "voyage-3-5",
});
@ -112,7 +119,7 @@ describe("buildAutoRouterTestTargets", () => {
it.each([[undefined], [""], [" "]])("adds no default target for %o", (defaultModel) => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
tiers: tierEntries(["gpt-4o-mini"]),
semanticMatchingEnabled: false,
embeddingModel: undefined,
defaultModel,

18
ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts
View file

@ -1,5 +1,3 @@
import { ComplexityTiers } from "./ComplexityRouterConfig";
export type AutoRouterTestMode = "chat" | "embedding";
export interface AutoRouterTestTarget {
@ -9,7 +7,8 @@ export interface AutoRouterTestTarget {
}
export interface BuildAutoRouterTestTargetsParams {
tiers: ComplexityTiers;
/** Ordered [tier name, model groups] entries of the active tier set. */
tiers: readonly (readonly [string, string[]])[];
semanticMatchingEnabled: boolean;
embeddingModel: string | undefined;
/** The resolved default model - see resolveComplexityDefaultModel. A live fallback destination,
@ -17,23 +16,14 @@ export interface BuildAutoRouterTestTargetsParams {
defaultModel?: string;
}
// Keys drive iteration order; `satisfies Record<keyof ComplexityTiers, null>` makes it a
// compile error to add a tier to ComplexityTiers without listing it here (and vice versa).
const TIER_ORDER = Object.keys({
SIMPLE: null,
MEDIUM: null,
COMPLEX: null,
REASONING: null,
} satisfies Record<keyof ComplexityTiers, null>) as (keyof ComplexityTiers)[];
export const buildAutoRouterTestTargets = ({
tiers,
semanticMatchingEnabled,
embeddingModel,
defaultModel,
}: BuildAutoRouterTestTargetsParams): AutoRouterTestTarget[] => {
const tieredByModel = TIER_ORDER.reduce<Record<string, string[]>>((acc, tier) => {
return (tiers[tier] ?? []).reduce((tierAcc, rawModel) => {
const tieredByModel = tiers.reduce<Record<string, string[]>>((acc, [tier, models]) => {
return models.reduce((tierAcc, rawModel) => {
const modelGroup = rawModel?.trim();
if (!modelGroup) return tierAcc;
return { ...tierAcc, [modelGroup]: [...(tierAcc[modelGroup] ?? []), tier] };

17
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -9,6 +9,7 @@ import {
hydrateTierLabels,
BuildComplexityRouterConfigParams,
} from "./build_complexity_router_config";
import { activeTierRows } from "./tier_rows";
const tiers = {
SIMPLE: ["gpt-4o-mini"],
@ -275,30 +276,30 @@ describe("buildComplexityRouterConfig", () => {
describe("getMissingTiersError", () => {
it("returns null when all four tiers have a model", () => {
expect(getMissingTiersError(tiers)).toBeNull();
expect(getMissingTiersError(activeTierRows({ tiers: tiers }))).toBeNull();
});
it("names the specific missing tier when only one is blank", () => {
expect(getMissingTiersError({ ...tiers, REASONING: [] })).toBe(
expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, REASONING: [] } }))).toBe(
"Select a model for the following tier(s): REASONING",
);
});
it("names multiple missing tiers in SIMPLE/MEDIUM/COMPLEX/REASONING order", () => {
expect(getMissingTiersError({ ...tiers, SIMPLE: [], REASONING: [] })).toBe(
expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: [], REASONING: [] } }))).toBe(
"Select a model for the following tier(s): SIMPLE, REASONING",
);
});
it("names all four tiers when none are filled", () => {
const noTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] };
expect(getMissingTiersError(noTiers)).toBe(
expect(getMissingTiersError(activeTierRows({ tiers: noTiers }))).toBe(
"Select a model for the following tier(s): SIMPLE, MEDIUM, COMPLEX, REASONING",
);
});
it("treats a tier with more than one model as filled", () => {
expect(getMissingTiersError({ ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] })).toBeNull();
expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] } }))).toBeNull();
});
});
@ -645,15 +646,15 @@ describe("getPlanModeTierError", () => {
const tiersWithEmptyComplex = { SIMPLE: ["m1"], MEDIUM: ["m1"], COMPLEX: [], REASONING: [] };
it("passes when the override is off", () => {
expect(getPlanModeTierError(undefined, tiersWithEmptyComplex)).toBeNull();
expect(getPlanModeTierError(undefined, activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull();
});
it("passes when the named tier has models", () => {
expect(getPlanModeTierError("MEDIUM", tiersWithEmptyComplex)).toBeNull();
expect(getPlanModeTierError("MEDIUM", activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull();
});
it("blocks a tier whose models were removed, which the backend would reject with a 400", () => {
expect(getPlanModeTierError("COMPLEX", tiersWithEmptyComplex)).toContain("COMPLEX");
expect(getPlanModeTierError("COMPLEX", activeTierRows({ tiers: tiersWithEmptyComplex }))).toContain("COMPLEX");
});
});

28
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -1,4 +1,5 @@
import { KeywordTierRule } from "./KeywordTierRules";
import { type TierRow, activeTierName, tierRowById } from "./tier_rows";
import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords";
import { TierModelParams, TierModelParamsByTier, serializeTierModelConfigs } from "./complexity_router_tiers";
import {
@ -10,6 +11,7 @@ import {
ComplexityTierLabels,
ComplexityTiers,
DimensionWeights,
TIER_KEYS,
TIER_DESCRIPTIONS,
TierBoundaries,
TokenThresholds,
@ -135,8 +137,6 @@ export interface ComplexityRouterConfigPayload {
tier_model_configs?: Record<string, { model_name: string; litellm_params: TierModelParams }[]>;
}
const TIER_KEYS: Array<keyof ComplexityTiers> = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"];
export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => {
const renamed = TIER_KEYS.map((tier) => [tier, tierLabels?.[tier]?.trim() ?? ""] as const).filter(
([tier, label]) => label !== "" && label !== TIER_DESCRIPTIONS[tier].label,
@ -171,26 +171,20 @@ export const getTierLabelsError = (tierLabels: ComplexityTierLabels | undefined)
return null;
};
// Requires all 4 tiers non-empty, so the create form can never reach the
// resolveComplexityDefaultModel(tiers, ...) === undefined case — MEDIUM (or SIMPLE) is always
// populated. The edit modal has no equivalent of this check (it allows saving with only some
// tiers filled), which is why it needs its own explicit `!defaultModel` guard after deriving —
// see edit_auto_router_modal.tsx's save handler. A future contributor copying this form's submit
// handler elsewhere should not assume the same guarantee holds without this check.
export const getMissingTiersError = (tiers: ComplexityTiers): string | null => {
const missing = TIER_KEYS.filter((tier) => tiers[tier].length === 0);
// Requires every active tier non-empty, so the create form can never reach the
// resolveComplexityDefaultModel === undefined case. The edit modal allows a partially filled
// set, which is why it keeps its own !defaultModel guard after deriving.
export const getMissingTiersError = (rows: readonly TierRow[]): string | null => {
const missing = rows.filter((row) => row.models.length === 0).map(activeTierName);
if (missing.length === 0) return null;
return `Select a model for the following tier(s): ${missing.join(", ")}`;
};
// The backend rejects a plan-mode floor naming a tier with no models. The create form's
// getMissingTiersError makes this unreachable there; the edit modal allows partially filled
// tiers, so both gates call this to keep the two forms symmetric.
export const getPlanModeTierError = (planModeMinTier: string | undefined, tiers: ComplexityTiers): string | null => {
export const getPlanModeTierError = (planModeMinTier: string | undefined, rows: readonly TierRow[]): string | null => {
if (!planModeMinTier) return null;
const models = tiers[planModeMinTier as keyof ComplexityTiers] ?? [];
if (models.length > 0) return null;
return `The plan-mode minimum tier (${planModeMinTier}) has no models. Add one or turn the override off.`;
const floor = tierRowById(rows, planModeMinTier);
if (floor && floor.models.length > 0) return null;
return `The plan-mode minimum tier (${floor ? activeTierName(floor) : planModeMinTier}) has no models. Add one or turn the override off.`;
};
export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => {

16
ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts
View file

@ -4,10 +4,10 @@ import {
hydrateTierModelParams,
normalizeTierModels,
pruneTierModelParams,
resolveComplexityDefaultModel,
serializeTierModelConfigs,
setTierModelReasoningEffort,
} from "./complexity_router_tiers";
import { resolveComplexityDefaultModel } from "./tier_rows";
import type { ComplexityTiers } from "./ComplexityRouterConfig";
@ -50,31 +50,31 @@ describe("resolveComplexityDefaultModel", () => {
const noTiers: ComplexityTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] };
it("derives from MEDIUM first when nothing is pinned", () => {
expect(resolveComplexityDefaultModel(tiers)).toBe("medium-model");
expect(resolveComplexityDefaultModel({ tiers: tiers })).toBe("medium-model");
});
it("falls back to SIMPLE when MEDIUM is empty", () => {
expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [] })).toBe("simple-model");
expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("simple-model");
});
it("derives nothing from COMPLEX or REASONING, which the backend never falls through to", () => {
expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [], SIMPLE: [] })).toBeUndefined();
expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [], SIMPLE: [] } })).toBeUndefined();
});
it("lets a pin beat the tiers rather than merely filling in for them", () => {
expect(resolveComplexityDefaultModel(tiers, "pinned-model")).toBe("pinned-model");
expect(resolveComplexityDefaultModel({ tiers: tiers }, "pinned-model")).toBe("pinned-model");
});
it("stands alone as the default when no tier holds a model", () => {
expect(resolveComplexityDefaultModel(noTiers, "pinned-model")).toBe("pinned-model");
expect(resolveComplexityDefaultModel({ tiers: noTiers }, "pinned-model")).toBe("pinned-model");
});
it.each([[""], [" "], [undefined]])("reads %o as no pin and goes back to the tiers", (pinned) => {
expect(resolveComplexityDefaultModel(tiers, pinned)).toBe("medium-model");
expect(resolveComplexityDefaultModel({ tiers: tiers }, pinned)).toBe("medium-model");
});
it("resolves to nothing when neither a pin nor a tier offers a model", () => {
expect(resolveComplexityDefaultModel(noTiers)).toBeUndefined();
expect(resolveComplexityDefaultModel({ tiers: noTiers })).toBeUndefined();
});
});

16
ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
View file

@ -1,5 +1,5 @@
import type { ComplexityTiers } from "./ComplexityRouterConfig";
import type { ComplexityTier } from "./KeywordTierRules";
import { TIER_ORDER } from "./tier_rows";
export type TierModelParams = Record<string, unknown>;
@ -80,13 +80,13 @@ export const hydrateTierModelParams = (
* tiers this editor does not render pass through rather than being dropped now the key is managed.
*/
export const serializeTierModelConfigs = (
tiers: ComplexityTiers,
tiers: Record<string, string[]>,
tierModelParams: TierModelParamsByTier | undefined,
): Record<string, { model_name: string; litellm_params: TierModelParams }[]> | undefined => {
if (tierModelParams === undefined) return undefined;
const serialized = Object.entries(tierModelParams)
.map(([tier, byModel]) => {
const selected = (TIER_ORDER as string[]).includes(tier) ? new Set(tiers[tier as ComplexityTier]) : undefined;
const selected = tier in tiers ? new Set(tiers[tier]) : undefined;
const entries = Object.entries(byModel)
.filter(([model, params]) => (selected === undefined || selected.has(model)) && Object.keys(params).length > 0)
.map(([model_name, litellm_params]) => ({ model_name, litellm_params }));
@ -126,14 +126,6 @@ export const pruneTierModelParams = (
return Object.keys(next).length > 0 ? next : undefined;
};
/**
* Mirrors `init_complexity_router_deployment` (litellm/router.py): an explicit pin wins, otherwise
* the default is `MEDIUM or SIMPLE`. Deriving past SIMPLE would name a model the backend never
* picks, and it raises rather than falling through to COMPLEX/REASONING.
*/
export const resolveComplexityDefaultModel = (tiers: ComplexityTiers, pinned?: string): string | undefined =>
pinned?.trim() || tiers.MEDIUM[0] || tiers.SIMPLE[0];
export const DEFAULT_TIER_LABELS: Record<ComplexityTier, string> = {
SIMPLE: "Simple",
MEDIUM: "Medium",
@ -141,8 +133,6 @@ export const DEFAULT_TIER_LABELS: Record<ComplexityTier, string> = {
REASONING: "Reasoning",
};
export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"];
export const tierOptions = (
tierLabels: Partial<Record<ComplexityTier, string>> | undefined,
): { value: ComplexityTier; label: string }[] =>

70
ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts Normal file
View file

@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
activeTierName,
activeTierRows,
isBuiltInTierName,
resolveComplexityDefaultModel,
sameTierIdentity,
tierRowById,
tierRowByName,
} from "./tier_rows";
const tiers = { SIMPLE: ["a"], MEDIUM: ["b"], COMPLEX: ["c"], REASONING: ["d"] };
describe("activeTierRows", () => {
it("reads the tier set as rows whose id is the canonical tier key, in severity order", () => {
expect(activeTierRows({ tiers })).toEqual([
{ id: "SIMPLE", name: "SIMPLE", models: ["a"] },
{ id: "MEDIUM", name: "MEDIUM", models: ["b"] },
{ id: "COMPLEX", name: "COMPLEX", models: ["c"] },
{ id: "REASONING", name: "REASONING", models: ["d"] },
]);
});
it("gives a tier with no models an empty pool rather than dropping the row", () => {
expect(activeTierRows({ tiers: { ...tiers, COMPLEX: [] } })[2]).toEqual({
id: "COMPLEX",
name: "COMPLEX",
models: [],
});
});
it("finds a row by id and by name", () => {
const rows = activeTierRows({ tiers });
expect(tierRowById(rows, "MEDIUM")?.models).toEqual(["b"]);
expect(tierRowById(rows, undefined)).toBeUndefined();
expect(tierRowByName(rows, " medium ")?.id).toBe("MEDIUM");
});
});
describe("sameTierIdentity", () => {
it.each([
["AUDIT", "audit", true],
["AUDIT", " audit ", true],
["AUDIT", "AUDITS", false],
])("compares %s and %s casefold, matching the backend's uniqueness rule", (left, right, expected) => {
expect(sameTierIdentity(left, right)).toBe(expected);
});
it("recognises the four built-in names regardless of case", () => {
expect(["SIMPLE", "medium", "Complex", "REASONING"].every(isBuiltInTierName)).toBe(true);
expect(isBuiltInTierName("SECURITY_REVIEW")).toBe(false);
});
it("trims a row name, since the backend matches fallback_tier and keyword rules exactly", () => {
expect(activeTierName({ id: "1", name: " AUDIT ", models: [] })).toBe("AUDIT");
});
});
describe("resolveComplexityDefaultModel", () => {
it("mirrors init_complexity_router_deployment: a pin wins, then MEDIUM, then SIMPLE", () => {
expect(resolveComplexityDefaultModel({ tiers }, "pinned")).toBe("pinned");
expect(resolveComplexityDefaultModel({ tiers })).toBe("b");
expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("a");
});
it("resolves to nothing rather than falling through to COMPLEX, which the backend never picks", () => {
expect(resolveComplexityDefaultModel({ tiers: { ...tiers, SIMPLE: [], MEDIUM: [] } })).toBeUndefined();
});
});

40
ui/litellm-dashboard/src/components/add_model/tier_rows.ts Normal file
View file

@ -0,0 +1,40 @@
import type { ComplexityTiers } from "./ComplexityRouterConfig";
import type { ComplexityTier } from "./KeywordTierRules";
export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"];
export interface TierRow {
id: string;
name: string;
models: string[];
}
export interface ActiveTierSet {
tiers: ComplexityTiers;
}
export const activeTierName = (row: TierRow): string => row.name.trim();
export const sameTierIdentity = (left: string, right: string): boolean =>
left.trim().toLowerCase() === right.trim().toLowerCase();
export const isBuiltInTierName = (name: string): boolean => TIER_ORDER.some((tier) => sameTierIdentity(tier, name));
// The only reader of the tier set. A row's id is the canonical tier key, so anything pointing into
// the set (the plan-mode floor, per-model params) points at a row rather than at a position.
export const activeTierRows = (value: ActiveTierSet): TierRow[] =>
TIER_ORDER.map((tier) => ({ id: tier, name: tier, models: value.tiers[tier] ?? [] }));
export const tierRowById = (rows: readonly TierRow[], id: string | undefined): TierRow | undefined =>
id === undefined ? undefined : rows.find((row) => row.id === id);
export const tierRowByName = (rows: readonly TierRow[], name: string): TierRow | undefined =>
rows.find((row) => sameTierIdentity(row.name, name));
// Mirrors init_complexity_router_deployment (litellm/router.py): a pin wins, then MEDIUM or SIMPLE
// looked up by exact name.
export const resolveComplexityDefaultModel = (value: ActiveTierSet, pinned?: string): string | undefined => {
const rows = activeTierRows(value);
const named = (name: string) => rows.find((row) => activeTierName(row) === name)?.models[0];
return pinned?.trim() || named("MEDIUM") || named("SIMPLE");
};

79
ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx
View file

@ -1,79 +0,0 @@
import { render, screen } from "@testing-library/react";
import { useState } from "react";
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
import { describe, it, expect, vi } from "vitest";
import DurationSelect from "./DurationSelect";
describe("DurationSelect", () => {
it("should render", () => {
render(<DurationSelect />);
expect(screen.getByRole("combobox")).toBeInTheDocument();
});
it("should render all three duration options", async () => {
const user = userEvent.setup();
render(<DurationSelect />);
const select = screen.getByRole("combobox");
await user.click(select);
expect(screen.getByText("Daily")).toBeInTheDocument();
expect(screen.getByText("Weekly")).toBeInTheDocument();
expect(screen.getByText("Monthly")).toBeInTheDocument();
const dailyLabel = screen.getByText("Daily");
const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel;
await user.click(dailyOption);
});
it("should apply className prop", () => {
render(<DurationSelect className="test-class" />);
const select = screen.getByRole("combobox");
expect(select.closest(".test-class")).toBeInTheDocument();
});
it("should call onChange when an option is selected", async () => {
const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
const onChange = vi.fn();
render(<DurationSelect onChange={onChange} />);
const select = screen.getByRole("combobox");
await user.click(select);
const dailyLabel = screen.getByText("Daily");
const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel;
await user.click(dailyOption);
expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object));
});
it("should accept and pass value prop to Select", () => {
render(<DurationSelect value="7d" />);
const select = screen.getByRole("combobox");
expect(select).toBeInTheDocument();
});
it.each([
["24h", "Daily"],
["7d", "Weekly"],
["30d", "Monthly"],
])("shows the human label on the trigger for %s", (value, label) => {
render(<DurationSelect value={value} />);
expect(screen.getByRole("combobox")).toHaveTextContent(label);
});
it("shows the human label on the trigger after the user picks an option", async () => {
const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
const Harness = () => {
const [value, setValue] = useState("24h");
return <DurationSelect value={value} onChange={setValue} />;
};
render(<Harness />);
await user.click(screen.getByRole("combobox"));
const monthly = screen.getByText("Monthly");
await user.click(monthly.closest('[role="option"]') ?? monthly);
expect(screen.getByRole("combobox")).toHaveTextContent("Monthly");
});
});

39
ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx
View file

@ -1,39 +0,0 @@
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
interface DurationSelectProps {
className?: string;
value?: string;
onChange?: (value: string, option: { value: string; label: string }) => void;
}
const DURATION_OPTIONS = [
{ value: "24h", label: "Daily" },
{ value: "7d", label: "Weekly" },
{ value: "30d", label: "Monthly" },
];
export default function DurationSelect({ className, value, onChange }: DurationSelectProps) {
return (
<Select
items={DURATION_OPTIONS}
value={value}
onValueChange={(nextValue) => {
const selectedOption = DURATION_OPTIONS.find((option) => option.value === nextValue);
if (selectedOption) {
onChange?.(selectedOption.value, selectedOption);
}
}}
>
<SelectTrigger className={className}>
<SelectValue placeholder="Select duration" />
</SelectTrigger>
<SelectContent>
{DURATION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
);
}

124
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
View file

@ -14,25 +14,21 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC
import { modelAvailableCall, modelPatchUpdateCall } from "../networking";
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
import RouterConfigBuilder from "../add_model/RouterConfigBuilder";
import {
hydrateTierModelParams,
normalizeTierModels,
resolveComplexityDefaultModel,
serializeTierModelConfigs,
} from "../add_model/complexity_router_tiers";
import { hydrateTierModelParams, normalizeTierModels } from "../add_model/complexity_router_tiers";
import { type ActiveTierSet, activeTierRows, resolveComplexityDefaultModel } from "../add_model/tier_rows";
import { isComplexityRouter } from "../add_model/auto_router_strategies";
import {
type BuildComplexityRouterConfigParams,
buildComplexityRouterConfig,
getKeywordTierRulesError,
getSemanticConfigError,
getPlanModeTierError,
getTierLabelsError,
hydrateTierLabels,
normalizeClassifierLlmConfig,
serializeTierLabels,
} from "../add_model/build_complexity_router_config";
import { KeywordTierRule } from "../add_model/KeywordTierRules";
import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching";
import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model/complexity_router_keywords";
import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords";
import {
hydrateDimensionWeights,
hydrateReasoningOverrideMinScore,
@ -46,7 +42,6 @@ import ComplexityRouterConfig, {
DEFAULT_SESSION_AFFINITY,
DEFAULT_DEPLOYMENT_AFFINITY,
DEFAULT_TIER_DISTANCE_PENALTY,
heuristicScoringRole,
} from "../add_model/ComplexityRouterConfig";
import {
Dialog,
@ -119,12 +114,12 @@ const toRecord = (value: unknown): Record<string, unknown> => {
export const hydratePinnedDefaultModel = (
storedConfigDefaultModel: unknown,
litellmParamsDefaultModel: string | null | undefined,
tiers: ComplexityTiers,
activeTiers: ActiveTierSet,
): string | undefined => {
if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) {
return storedConfigDefaultModel;
}
const tierDerived = resolveComplexityDefaultModel(tiers);
const tierDerived = resolveComplexityDefaultModel(activeTiers);
const externalOverride = litellmParamsDefaultModel?.trim();
return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined;
};
@ -148,73 +143,48 @@ export const buildUpdatedComplexityRouterConfig = (
if (keywordMatching !== undefined && KEYWORD_MATCHING_KEYS.has(key)) return true;
return customTechnicalKeywords !== undefined && key === "custom_technical_keywords";
};
const preservedConfig = Object.fromEntries(Object.entries(toRecord(storedConfig)).filter(([key]) => !isManaged(key)));
const adaptiveEligible = value.adaptive_eligible ?? "all";
const storedKeywordRules = keywordMatching ? serializeKeywordTierRules(keywordMatching.keywordTierRules) : [];
const serializedTierLabels = serializeTierLabels(value.tier_labels);
const scorerRuns = heuristicScoringRole(value) !== "never";
const serializedTierModelConfigs = serializeTierModelConfigs(value.tiers, value.tier_model_params);
const builderParams: BuildComplexityRouterConfigParams = {
tiers: value.tiers,
defaultModel: value.default_model,
planModeMinTier: value.plan_mode_min_tier,
tierLabels: value.tier_labels,
classifierType: value.classifier_type,
classifierLlmConfig: value.classifier_llm_config,
classifierContextWindowSize: value.classifier_context_window_size,
classifierContextBudgetChars: value.classifier_context_budget_chars,
classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
classifierFallback: value.classifier_fallback,
sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
customTechnicalKeywords: customTechnicalKeywords ?? [],
keywordTierRules: keywordMatching?.keywordTierRules ?? [],
semanticMatchingEnabled: keywordMatching?.semanticMatchingEnabled ?? false,
embeddingModel: keywordMatching?.embeddingModel,
matchThreshold: keywordMatching?.matchThreshold ?? DEFAULT_MATCH_THRESHOLD,
escalationKeywords: keywordMatching?.escalationKeywords ?? [],
adaptive: value.adaptive ?? false,
adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
adaptiveEligible: value.adaptive_eligible ?? "all",
returnRawModelName: value.return_raw_model_name ?? false,
tierBoundaries: value.tier_boundaries,
tokenThresholds: value.token_thresholds,
dimensionWeights: value.dimension_weights,
reasoningOverrideMinScore: value.reasoning_override_min_score,
tierModelParams: value.tier_model_params,
};
const built = buildComplexityRouterConfig(builderParams);
// Keys this call does not own stay as the stored config left them.
const unowned: readonly string[] = [
...(keywordMatching === undefined ? KEYWORD_MATCHING_KEYS : []),
...(customTechnicalKeywords === undefined ? ["custom_technical_keywords"] : []),
];
return {
...preservedConfig,
tiers: value.tiers,
...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }),
...(value.default_model?.trim() && { default_model: value.default_model }),
...(value.plan_mode_min_tier?.trim() && { plan_mode_min_tier: value.plan_mode_min_tier }),
...(serializedTierLabels && { tier_labels: serializedTierLabels }),
classifier_type: value.classifier_type,
...(value.classifier_type === "llm" && value.classifier_llm_config
? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) }
: {}),
...(value.classifier_type === "llm" &&
value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }),
...(value.classifier_type === "llm" &&
value.classifier_context_window_size !== undefined && {
classifier_context_window_size: value.classifier_context_window_size,
}),
...(value.classifier_type === "llm" &&
value.classifier_context_budget_chars !== undefined && {
classifier_context_budget_chars: value.classifier_context_budget_chars,
}),
...(value.classifier_type === "llm" &&
value.classifier_context_include_assistant_turns !== undefined && {
classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns,
}),
session_affinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
deployment_affinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
...(customTechnicalKeywords &&
customTechnicalKeywords.length > 0 && {
custom_technical_keywords: customTechnicalKeywords,
}),
...(value.adaptive && {
adaptive: true,
adaptive_weights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
...(adaptiveEligible === "all" && {
tier_distance_penalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
}),
adaptive_eligible: adaptiveEligible,
}),
...(value.return_raw_model_name && { return_raw_model_name: true }),
...(keywordMatching && {
// Mirrors buildComplexityRouterConfig: the key only when there is a rule to write,
// escalation keywords always, semantic trio only when on.
...(storedKeywordRules.length > 0 && { keyword_tier_rules: storedKeywordRules }),
escalation_keywords: keywordMatching.escalationKeywords.map((k) => k.trim()).filter(Boolean),
...(keywordMatching.semanticMatchingEnabled && {
semantic_keyword_matching: true,
embedding_model: keywordMatching.embeddingModel,
match_threshold: keywordMatching.matchThreshold,
}),
}),
...(scorerRuns && value.tier_boundaries !== undefined && { tier_boundaries: value.tier_boundaries }),
...(scorerRuns && value.token_thresholds !== undefined && { token_thresholds: value.token_thresholds }),
...(scorerRuns && value.dimension_weights !== undefined && { dimension_weights: value.dimension_weights }),
...(scorerRuns &&
value.reasoning_override_min_score !== undefined && {
reasoning_override_min_score: value.reasoning_override_min_score,
}),
...Object.fromEntries(Object.entries(built).filter(([key]) => !unowned.includes(key))),
};
};
@ -297,7 +267,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
? "Please select at least one model for a complexity tier"
: null) ??
getTierLabelsError(complexityRouterConfig.tier_labels) ??
getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, complexityRouterConfig.tiers) ??
getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
getKeywordTierRulesError(keywordTierRules);
useEffect(() => {
@ -355,7 +325,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
default_model: hydratePinnedDefaultModel(
parsedConfig.default_model,
modelData.litellm_params?.complexity_router_default_model,
hydratedTiers,
{ tiers: hydratedTiers },
),
plan_mode_min_tier:
typeof parsedConfig.plan_mode_min_tier === "string" && parsedConfig.plan_mode_min_tier.trim() !== ""
@ -486,7 +456,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
// build_complexity_router_config.ts for why create never can). init_complexity_router_deployment
// raises in that case (litellm/router.py), so block it rather than saving a router that
// fails at init.
const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model);
const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model);
if (!defaultModel) {
setShowValidationErrors(true);
toast.fromError(

14
ui/litellm-dashboard/src/components/model_info_view.tsx
View file

@ -16,7 +16,7 @@ import { stripMaskedSecrets } from "../utils/maskedSecretUtils";
import { truncateString } from "../utils/textUtils";
import AutoRouterConnectionTest from "./add_model/auto_router_connection_test";
import { AutoRouterTestTarget, buildAutoRouterTestTargets } from "./add_model/build_auto_router_test_targets";
import { normalizeTierModels, resolveComplexityDefaultModel } from "./add_model/complexity_router_tiers";
import { normalizeTierModels } from "./add_model/complexity_router_tiers";
import {
hasAutoRouterEditor,
isAutoRouterDeployment,
@ -91,12 +91,10 @@ const buildComplexityRouterTestTargets = (
config = rawConfig;
}
const tiers = {
SIMPLE: normalizeTierModels(config.tiers?.SIMPLE),
MEDIUM: normalizeTierModels(config.tiers?.MEDIUM),
COMPLEX: normalizeTierModels(config.tiers?.COMPLEX),
REASONING: normalizeTierModels(config.tiers?.REASONING),
};
const tiers: [string, string[]][] =
config.tiers && typeof config.tiers === "object"
? Object.entries(config.tiers).map(([tier, models]) => [tier, normalizeTierModels(models)])
: [];
// Mirrors init_complexity_router_deployment (litellm/router.py): litellm_params wins, otherwise
// pure tier-derivation. complexity_router_config.default_model is a UI-only marker the backend
@ -108,7 +106,7 @@ const buildComplexityRouterTestTargets = (
tiers,
semanticMatchingEnabled: Boolean(config.semantic_keyword_matching),
embeddingModel: config.embedding_model,
defaultModel: resolveComplexityDefaultModel(tiers, effectiveDefaultModel),
defaultModel: effectiveDefaultModel,
};
return buildAutoRouterTestTargets(testTargetParams);
};

54
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -4,7 +4,7 @@ import { fireEvent, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import TeamInfoView from "./TeamInfo";
import TeamInfoView, { type TeamData } from "./TeamInfo";
const authState = vi.hoisted(() => ({ userRole: "Admin" }));
@ -1613,10 +1613,18 @@ describe("TeamInfoView - which team member fields reach the update payload depen
vi.clearAllMocks();
});
const openEditor = async (user: ReturnType<typeof userEvent.setup>) => {
const openEditor = async (
user: ReturnType<typeof userEvent.setup>,
teamMemberBudgetTable: TeamData["team_info"]["team_member_budget_table"] = {
max_budget: 42,
budget_duration: "30d",
tpm_limit: 11,
rpm_limit: 22,
},
) => {
vi.mocked(networking.teamInfoCall).mockResolvedValue(
createMockTeamData({
team_member_budget_table: { max_budget: 42, budget_duration: "30d", tpm_limit: 11, rpm_limit: 22 },
team_member_budget_table: teamMemberBudgetTable,
default_team_member_models: ["gpt-4"],
}),
);
@ -1667,6 +1675,46 @@ describe("TeamInfoView - which team member fields reach the update payload depen
expect(payload.default_team_member_models).toEqual(["gpt-4"]);
});
it("sends a null team_member_budget_duration when Default Budget Duration is set to never reset", async () => {
const user = userEvent.setup({ delay: null });
await openEditor(user);
await user.click(screen.getByText("Team Member Settings"));
await screen.findByLabelText("Default Budget (USD)");
await chooseSelectOption(user, screen.getByLabelText("Default Budget Duration"), "Never resets");
const payload = await save(user);
expect(payload.team_member_budget_duration).toBeNull();
expect(payload.team_member_budget).toBe(42);
expect(JSON.stringify(payload)).toContain('"team_member_budget_duration":null');
});
it("shows Never resets for a stored member budget whose duration is null", async () => {
const user = userEvent.setup({ delay: null });
await openEditor(user, { max_budget: 42, budget_duration: null, tpm_limit: null, rpm_limit: null });
await user.click(screen.getByText("Team Member Settings"));
expect(await screen.findByLabelText("Default Budget Duration")).toHaveTextContent("Never resets");
});
it("omits team_member_budget_duration when the dropdown is left untouched on a team with no member budget", async () => {
const user = userEvent.setup({ delay: null });
await openEditor(user, null);
await user.click(screen.getByText("Team Member Settings"));
const durationSelect = await screen.findByLabelText("Default Budget Duration");
expect(durationSelect).toHaveTextContent("Inherit team reset period");
expect(durationSelect).not.toHaveTextContent("Never resets");
await user.type(screen.getByLabelText("Default Budget (USD)"), "100");
const payload = await save(user);
expect(payload.team_member_budget).toBe(100);
expect(JSON.parse(JSON.stringify(payload))).not.toHaveProperty("team_member_budget_duration");
});
it("omits object_permission.search_tools while Search Tool Settings is closed", async () => {
const user = userEvent.setup({ delay: null });
await openEditor(user);

15
ui/litellm-dashboard/src/components/team/TeamInfo.tsx
View file

@ -48,7 +48,7 @@ import { z } from "zod/v4";
import GuardrailsSelect from "./GuardrailsSelect";
import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils";
import AccessGroupSelector from "../common_components/AccessGroupSelector";
import BudgetDurationDropdown from "../common_components/budget_duration_dropdown";
import BudgetDurationDropdown, { NEVER_RESETS_BUDGET_DURATION } from "../common_components/budget_duration_dropdown";
import {
computeTeamModelBadges,
normalizeTeamModelSelection,
@ -64,7 +64,6 @@ import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMeta
import ModelAliasManager from "../common_components/ModelAliasManager";
import AgentSelector from "../agent_management/AgentSelector";
import DeleteResourceModal from "../common_components/DeleteResourceModal";
import DurationSelect from "../common_components/DurationSelect";
import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector";
import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key";
import GuardrailSettingsView from "../GuardrailSettingsView";
@ -169,7 +168,7 @@ export interface TeamData {
object_permission?: ObjectPermission | null;
team_member_budget_table: {
max_budget: number;
budget_duration: string;
budget_duration: string | null;
tpm_limit: number | null;
rpm_limit: number | null;
} | null;
@ -1256,7 +1255,15 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
name="team_member_budget_duration"
label="Default Budget Duration"
>
{({ value, onChange }) => <DurationSelect value={value ?? undefined} onChange={onChange} />}
{({ id, value, onChange }) => (
<BudgetDurationDropdown
id={id}
showNeverResets
placeholder="Inherit team reset period"
value={value === null ? NEVER_RESETS_BUDGET_DURATION : value}
onChange={(next) => onChange(next === NEVER_RESETS_BUDGET_DURATION ? null : next)}
/>
)}
</FormField>
<FormField
control={form.control}

13
ui/litellm-dashboard/src/lib/autorouter_presets.ts
View file

@ -4,7 +4,6 @@ import {
} from "@/components/add_model/build_complexity_router_config";
import {
ComplexityRouterConfigValue,
ComplexityTiers,
ClassifierType,
ClassifierLLMConfig,
DEFAULT_SESSION_AFFINITY,
@ -43,15 +42,7 @@ export const getRequiredModels = (
config: Pick<ComplexityRouterConfigPayload, "tiers" | "classifier_llm_config" | "embedding_model" | "default_model">,
): Set<string> => {
const { tiers, classifier_llm_config: classifier, embedding_model: embedding, default_model: pinned } = config;
const models = [
...tiers.SIMPLE,
...tiers.MEDIUM,
...tiers.COMPLEX,
...tiers.REASONING,
classifier?.model,
embedding,
pinned,
];
const models = [...Object.values(tiers).flat(), classifier?.model, embedding, pinned];
// Boolean(), not != null: an empty-string placeholder (e.g. classifier_llm_config seeded before a
// model is chosen) is never a real model reference either.
return new Set(models.filter((model): model is string => Boolean(model)));
@ -191,7 +182,7 @@ export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability:
// effect would block submit for a model that was never going to be submitted.
export const getReferencedModelsError = (
params: {
tiers: ComplexityTiers;
tiers: ComplexityRouterConfigPayload["tiers"];
classifierType: ClassifierType;
classifierLlmConfig: ClassifierLLMConfig | undefined;
semanticMatchingEnabled: boolean;

6
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -11029,11 +11029,9 @@ export interface paths {
* -d '{
* "prompt_id": "my_prompt",
* "litellm_params": {
* "prompt_id": "json_prompt",
* "prompt_id": "my_prompt",
* "prompt_integration": "dotprompt",
* ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED
* "prompt_directory": "/path/to/dotprompt/folder",
* "prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}}
* "prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}
* },
* "prompt_info": {
* "prompt_type": "config"