diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6536941a094..4cd292b2416 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -794,14 +794,27 @@ def _select_model_name_for_cost_calc( and custom_llm_provider is not None and not _model_contains_known_llm_provider(return_model) ): # add provider prefix if not already present, to match model_cost - if region_name is not None: - return_model = f"{custom_llm_provider}/{region_name}/{return_model}" - else: - return_model = f"{custom_llm_provider}/{return_model}" + provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}" + return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name) return return_model +def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: + """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the + registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already + resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069).""" + segments: Final = model.split("/") + if "/".join(segments[1:]) in litellm.model_cost: + return model + head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1 + head: Final = "/".join(segments[:head_len]) + tail: Final = segments[head_len:] + strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail)) + candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1)) + return next((candidate for candidate in candidates if candidate in litellm.model_cost), model) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _model_contains_known_llm_provider(model: str) -> bool: """ diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 07d83bc34d5..1188bce27da 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom if dotprompt_content and not prompt_data and not prompt_file: prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content) + from .prompt_manager import strip_version_suffix + + registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id + try: dot_prompt_manager: Final = DotpromptManager( prompt_directory=prompt_directory, prompt_data=prompt_data, prompt_file=prompt_file, - prompt_id=prompt_id, + prompt_id=registration_prompt_id, ) return dot_prompt_manager diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index e5e868f0523..0f45e926c30 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement): if prompt_id is None: return False try: - return prompt_id in self.prompt_manager.list_prompts() + return self.prompt_manager.get_prompt(prompt_id) is not None except Exception: # If there's any error accessing prompts, don't run prompt management return False diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 46750ed9799..a0d5be71392 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment +def strip_version_suffix(prompt_id: str) -> str | None: + base, separator, version = prompt_id.rpartition(".v") + if separator and base and version.isdigit(): + return base + return None + + class PromptTemplate: """Represents a single prompt template with metadata and content.""" @@ -124,11 +131,13 @@ class PromptManager: "content": "template content", "metadata": {"model": "gpt-4", "temperature": 0.7, ...} } + prompt_id - """ - if prompt_id: - prompt_data = {prompt_id: prompt_data} - for prompt_id, prompt_info in prompt_data.items(): + A dict carrying a "content" key is a single flat template registered under + prompt_id; anything else is treated as already keyed by template ID. + """ + keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data + + for template_id, prompt_info in keyed_prompts.items(): try: content = prompt_info.get("content", "") metadata = prompt_info.get("metadata", {}) @@ -136,11 +145,11 @@ class PromptManager: template = PromptTemplate( content=content, metadata=metadata, - template_id=prompt_id, + template_id=template_id, ) - self.prompts[prompt_id] = template + self.prompts[template_id] = template except Exception: - # Optional: print(f"Error loading prompt from JSON: {prompt_id}") + # Optional: print(f"Error loading prompt from JSON: {template_id}") pass def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate: @@ -272,8 +281,12 @@ class PromptManager: if versioned_id in self.prompts: return self.prompts[versioned_id] - # Fall back to base prompt_id - return self.prompts.get(prompt_id) + direct_match: Final = self.prompts.get(prompt_id) + if direct_match is not None: + return direct_match + + base_prompt_id: Final = strip_version_suffix(prompt_id) + return self.prompts.get(base_prompt_id) if base_prompt_id else None def list_prompts(self) -> list[str]: """Get a list of all available prompt IDs.""" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3c799bcaa40..0f33310c734 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1428,7 +1428,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1465,7 +1465,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1502,7 +1502,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1539,7 +1539,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2933,7 +2933,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-10-19", @@ -2956,7 +2957,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", @@ -2987,7 +2989,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", @@ -3018,7 +3021,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { "supports_mid_conversation_system": true, @@ -3050,7 +3054,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5": { "supports_mid_conversation_system": true, @@ -3113,7 +3118,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-opus-4-1": { "deprecation_date": "2026-08-05", @@ -3135,7 +3141,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", @@ -3157,7 +3164,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-5": { "supports_mid_conversation_system": true, @@ -3188,7 +3196,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", @@ -3214,7 +3223,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -14724,7 +14734,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14746,7 +14757,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-1": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14768,7 +14780,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-5": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14791,7 +14804,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-6": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14814,7 +14828,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-7": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14916,7 +14931,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14960,7 +14976,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14983,7 +15000,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-5": { "cache_creation_input_token_cost": 3.74997e-06, @@ -33899,7 +33917,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -33919,7 +33938,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -33942,7 +33962,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, @@ -33968,7 +33989,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -33987,7 +34009,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -34008,7 +34031,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -34031,7 +34055,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -34049,7 +34074,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -34072,7 +34098,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "openrouter/anthropic/claude-opus-5": { "prompt_cache_min_tokens": 512, @@ -36539,7 +36566,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 4096 }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, @@ -36621,7 +36649,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/deepseek-ai/deepseek-v3": { "input_cost_per_token": 1.45e-06, @@ -36696,7 +36725,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, @@ -39725,7 +39755,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, @@ -39744,7 +39775,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.1": { "cache_creation_input_token_cost": 1.875e-05, @@ -39763,7 +39795,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -39783,7 +39816,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -39805,7 +39839,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -39824,7 +39859,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { "cache_creation_input_token_cost": 3.75e-06, @@ -39842,7 +39878,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, @@ -41207,7 +41244,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -41241,7 +41279,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -50152,7 +50191,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -50169,7 +50209,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { "max_tokens": 16384, @@ -50184,7 +50225,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { "max_tokens": 16384, @@ -50200,7 +50242,8 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_reasoning": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { "max_tokens": 16384, @@ -50215,7 +50258,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { "max_tokens": 16384, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 026a02d6b1d..0e3cbbfd560 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -23538,7 +23538,7 @@ "paths": { "/prompts": { "post": { - "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", + "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_data\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", "operationId": "create_prompt_prompts_post", "requestBody": { "content": { diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f2327b18914..c6d7975b75e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -14,6 +14,7 @@ import json import math import traceback from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protocol, TypeVar, cast @@ -494,8 +495,13 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: - """Create team member budget table with provided limits""" + """Create team member budget table with provided limits. + + The team's own reset period is only inherited when the caller left the + member duration out, so an explicit null means "never resets". + """ from litellm.proxy._types import BudgetNewRequest from litellm.proxy.management_endpoints.budget_management_endpoints import ( new_budget, @@ -509,7 +515,11 @@ class TeamMemberBudgetHandler: # Create budget request with all provided limits budget_request: Final = BudgetNewRequest( budget_id=budget_id, - budget_duration=data.budget_duration or team_member_budget_duration, + budget_duration=( + team_member_budget_duration + if "team_member_budget_duration" in explicitly_set_fields + else data.budget_duration or team_member_budget_duration + ), ) if team_member_budget is not None: @@ -545,8 +555,13 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: - """Upsert team member budget table with provided limits""" + """Upsert team member budget table with provided limits. + + A field the caller explicitly sent as null is written as null, so a + team can keep a member budget while dropping its reset period. + """ from litellm.proxy._types import BudgetNewRequest from litellm.proxy.management_endpoints.budget_management_endpoints import ( update_budget, @@ -560,14 +575,16 @@ class TeamMemberBudgetHandler: # Budget exists - create update request with only provided values budget_request: Final = BudgetNewRequest(budget_id=team_member_budget_id) - if team_member_budget is not None: + if team_member_budget is not None or "team_member_budget" in explicitly_set_fields: budget_request.max_budget = team_member_budget - if team_member_rpm_limit is not None: + if team_member_rpm_limit is not None or "team_member_rpm_limit" in explicitly_set_fields: budget_request.rpm_limit = team_member_rpm_limit - if team_member_tpm_limit is not None: + if team_member_tpm_limit is not None or "team_member_tpm_limit" in explicitly_set_fields: budget_request.tpm_limit = team_member_tpm_limit - if team_member_budget_duration is not None: + if team_member_budget_duration is not None or "team_member_budget_duration" in explicitly_set_fields: budget_request.budget_duration = team_member_budget_duration + if team_member_budget_duration is None: + budget_request.budget_reset_at = None budget_row: Final = await _as_budget_write(update_budget)( budget_obj=budget_request, @@ -593,6 +610,7 @@ class TeamMemberBudgetHandler: team_member_rpm_limit=team_member_rpm_limit, team_member_tpm_limit=team_member_tpm_limit, team_member_budget_duration=team_member_budget_duration, + explicitly_set_fields=explicitly_set_fields, ) # Remove team member fields from updated_kv @@ -1479,6 +1497,7 @@ async def new_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + explicitly_set_fields=data.model_fields_set, ) ## ADD TO TEAM TABLE @@ -2184,6 +2203,7 @@ async def update_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + explicitly_set_fields=_team_member_fields_in_request, ) # Backfill team_memberships for members who joined before the # budget was configured — they won't have a membership row yet. diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index a289ed7cbfb..e90adc163e2 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -323,6 +323,7 @@ def create_versioned_prompt_spec(db_prompt: _PromptRow) -> PromptSpec: prompt_info=prompt_info, created_at=row.created_at, updated_at=row.updated_at, + version=row.version, environment=row.environment, created_by=row.created_by, ) @@ -334,6 +335,21 @@ class Prompt(BaseModel): prompt_info: PromptInfo | None = None +AMBIGUOUS_PROMPT_DATA_ERROR: Final = ( + "litellm_params.prompt_id cannot be combined with prompt_data keyed by template name. " + 'Send a flat template, prompt_data={"content": "...", "metadata": {...}}, together with litellm_params.prompt_id, ' + 'or send prompt_data={"": {"content": "...", "metadata": {...}}} without litellm_params.prompt_id.' +) + + +def is_ambiguous_keyed_prompt_data(litellm_params: PromptLiteLLMParams) -> bool: + extra_fields: Final = litellm_params.model_extra or {} + prompt_data: Final = extra_fields.get("prompt_data") + if not litellm_params.prompt_id or not isinstance(prompt_data, dict): + return False + return bool(prompt_data) and "content" not in prompt_data + + class PatchPromptRequest(BaseModel): litellm_params: PromptLiteLLMParams | None = None prompt_info: PromptInfo | None = None @@ -737,11 +753,9 @@ async def create_prompt( -d '{ "prompt_id": "my_prompt", "litellm_params": { - "prompt_id": "json_prompt", + "prompt_id": "my_prompt", "prompt_integration": "dotprompt", - ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED - "prompt_directory": "/path/to/dotprompt/folder", - "prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}} + "prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}} }, "prompt_info": { "prompt_type": "config" @@ -763,6 +777,9 @@ async def create_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Extract environment from request environment: Final = ( @@ -857,6 +874,9 @@ async def update_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success") base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) @@ -1086,6 +1106,9 @@ async def patch_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if request.litellm_params is not None and is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Resolve the target row: find the latest version in the given environment base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index 695bdabfe83..46caa30fdab 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -147,6 +147,9 @@ class InMemoryPromptRegistry: prompt_info=prompt.prompt_info or PromptInfo(prompt_type="config"), created_at=prompt.created_at, updated_at=prompt.updated_at, + version=prompt.version, + environment=prompt.environment, + created_by=prompt.created_by, ) # store references to the prompt in memory diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e55b254ab8e..9a3905da950 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6268,7 +6268,14 @@ class ProxyConfig: ): from litellm.utils import _update_dictionary - combined_router_settings = _update_dictionary(config_router_settings, db_router_settings.param_value) + db_overlay_deferring_empty_lists_to_config: Final = { + k: v + for k, v in db_router_settings.param_value.items() + if not (k in config_router_settings and isinstance(v, list) and len(v) == 0) + } + combined_router_settings = _update_dictionary( + config_router_settings, db_overlay_deferring_empty_lists_to_config + ) elif config_router_settings is not None and isinstance(config_router_settings, dict): combined_router_settings = config_router_settings elif db_router_settings is not None and isinstance(db_router_settings.param_value, dict): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3c799bcaa40..0f33310c734 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1428,7 +1428,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1465,7 +1465,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1502,7 +1502,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1539,7 +1539,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2933,7 +2933,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-10-19", @@ -2956,7 +2957,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", @@ -2987,7 +2989,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", @@ -3018,7 +3021,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { "supports_mid_conversation_system": true, @@ -3050,7 +3054,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5": { "supports_mid_conversation_system": true, @@ -3113,7 +3118,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-opus-4-1": { "deprecation_date": "2026-08-05", @@ -3135,7 +3141,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", @@ -3157,7 +3164,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-5": { "supports_mid_conversation_system": true, @@ -3188,7 +3196,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", @@ -3214,7 +3223,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -14724,7 +14734,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14746,7 +14757,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-1": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14768,7 +14780,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-5": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14791,7 +14804,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-6": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14814,7 +14828,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-7": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14916,7 +14931,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14960,7 +14976,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14983,7 +15000,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-5": { "cache_creation_input_token_cost": 3.74997e-06, @@ -33899,7 +33917,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -33919,7 +33938,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -33942,7 +33962,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, @@ -33968,7 +33989,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -33987,7 +34009,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -34008,7 +34031,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -34031,7 +34055,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -34049,7 +34074,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -34072,7 +34098,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "openrouter/anthropic/claude-opus-5": { "prompt_cache_min_tokens": 512, @@ -36539,7 +36566,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 4096 }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, @@ -36621,7 +36649,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/deepseek-ai/deepseek-v3": { "input_cost_per_token": 1.45e-06, @@ -36696,7 +36725,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, @@ -39725,7 +39755,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, @@ -39744,7 +39775,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.1": { "cache_creation_input_token_cost": 1.875e-05, @@ -39763,7 +39795,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -39783,7 +39816,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -39805,7 +39839,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -39824,7 +39859,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { "cache_creation_input_token_cost": 3.75e-06, @@ -39842,7 +39878,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, @@ -41207,7 +41244,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -41241,7 +41279,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -50152,7 +50191,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -50169,7 +50209,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { "max_tokens": 16384, @@ -50184,7 +50225,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { "max_tokens": 16384, @@ -50200,7 +50242,8 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_reasoning": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { "max_tokens": 16384, @@ -50215,7 +50258,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { "max_tokens": 16384, diff --git a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py index b92ed13302e..3c10e0db8d7 100644 --- a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py +++ b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py @@ -577,3 +577,83 @@ async def test_dotprompt_with_prompt_version(): ) assert "Version 2:" in v2_rendered assert "Test v2" in v2_rendered + + +def test_keyed_prompt_data_with_prompt_id_keeps_real_content(): + prompt_data = { + "json_prompt": { + "content": "You are a pirate. Begin every reply with AHOY.", + "metadata": {"model": "gpt-4o-mini"}, + } + } + + manager = PromptManager(prompt_data=prompt_data, prompt_id="agent-prompt") + + template = manager.get_prompt("json_prompt") + assert template is not None + assert template.content == "You are a pirate. Begin every reply with AHOY." + assert template.model == "gpt-4o-mini" + assert "agent-prompt" not in manager.prompts + + +def test_flat_prompt_data_with_prompt_id_registers_under_prompt_id(): + manager = PromptManager( + prompt_data={"content": "Hello {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + prompt_id="flat-prompt", + ) + + template = manager.get_prompt("flat-prompt") + assert template is not None + assert template.content == "Hello {{name}}" + assert manager.render("flat-prompt", {"name": "world"}) == "Hello world" + + +def test_get_prompt_falls_back_to_base_id_for_versioned_id(): + manager = PromptManager( + prompt_data={"content": "Hi", "metadata": {}}, + prompt_id="my-prompt", + ) + + assert manager.get_prompt("my-prompt.v1") is not None + assert manager.get_prompt("my-prompt.v12") is not None + assert manager.get_prompt("my-prompt.vx") is None + assert manager.get_prompt("other-prompt.v1") is None + + +def test_should_run_prompt_management_accepts_versioned_id(): + from litellm.integrations.dotprompt import DotpromptManager + + dotprompt_manager = DotpromptManager( + prompt_data={"content": "Hi", "metadata": {}}, + prompt_id="versioned-prompt", + ) + + assert dotprompt_manager.should_run_prompt_management("versioned-prompt", None, {}) is True + assert dotprompt_manager.should_run_prompt_management("versioned-prompt.v1", None, {}) is True + assert dotprompt_manager.should_run_prompt_management("missing-prompt", None, {}) is False + + +def test_prompt_initializer_registers_flat_db_prompt_under_base_id(): + from litellm.integrations.dotprompt import DotpromptManager, prompt_initializer + from litellm.types.prompts.init_prompts import ( + PromptInfo, + PromptLiteLLMParams, + PromptSpec, + ) + + litellm_params = PromptLiteLLMParams( + prompt_integration="dotprompt", + prompt_data={"content": "AHOY {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + ) + prompt_spec = PromptSpec( + prompt_id="agent-prompt.v1", + litellm_params=litellm_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + + dotprompt_manager = prompt_initializer(litellm_params, prompt_spec) + + assert isinstance(dotprompt_manager, DotpromptManager) + template = dotprompt_manager.prompt_manager.get_prompt("agent-prompt") + assert template is not None + assert template.content == "AHOY {{name}}" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 177c71eeae7..18d1aa949a8 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2757,3 +2757,176 @@ def test_video_generation_with_input_reference_keeps_file_multipart(): "seconds": "4", } assert result.status == "queued" + + +AZURE_AI_BASE = "https://myfoundry.services.ai.azure.com" +AZURE_AI_CHAT_COMPLETIONS_URL = f"{AZURE_AI_BASE}/models/chat/completions" + +def _a_tool_with_an_unsupported_field() -> dict: + return { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, + "strict": True, + } + +A_COMPLETION = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "grok-3", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + +TOOL_LEVEL_REJECTION = "Extra inputs are not permitted: tools[0].strict" +UNRELATED_REJECTION = "Extra inputs are not permitted: temperature" +A_REJECTION_THE_PROVIDER_CANNOT_FIX = "The model is not available in this region" + + +class _RecordedAzureAI: + def __init__(self, responses: list[httpx.Response]) -> None: + self._responses = responses + self.bodies: list[dict] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.bodies.append(json.loads(request.content)) + return self._responses[min(len(self.bodies) - 1, len(self._responses) - 1)] + + +@pytest.fixture +def httpx_transport(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +def _rejection(message: str) -> httpx.Response: + return httpx.Response(422, json={"error": {"message": message}}) + + +def _call_azure_ai(recorder: _RecordedAzureAI, **overrides): + import respx + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + return litellm.completion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + **overrides, + ) + + +def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried(): + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + response = _call_azure_ai(recorder) + + assert len(recorder.bodies) == 2 + assert recorder.bodies[0]["tools"][0]["strict"] is True + assert "strict" not in recorder.bodies[1]["tools"][0] + assert response.choices[0].message.content == "sent" + + +def test_the_retry_changes_only_the_field_the_provider_named(): + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + _call_azure_ai(recorder) + + first, second = recorder.bodies + assert second["messages"] == first["messages"] + assert second["model"] == first["model"] + assert second["tools"][0]["function"] == first["tools"][0]["function"] + + +def test_a_provider_that_keeps_rejecting_is_not_retried_forever(): + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)]) + + with pytest.raises(litellm.BadRequestError) as raised: + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 2 + assert raised.value.status_code == 422 + + +def test_a_rejection_the_provider_cannot_fix_is_not_retried_at_all(): + recorder = _RecordedAzureAI([_rejection(A_REJECTION_THE_PROVIDER_CANNOT_FIX)]) + + with pytest.raises(litellm.BadRequestError): + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 1 + + +def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was_asked_for(): + recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION)]) + + with pytest.raises(litellm.BadRequestError): + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 1 + + +def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for(): + recorder = _RecordedAzureAI( + [_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + response = _call_azure_ai(recorder, drop_params=True) + + assert len(recorder.bodies) == 2 + assert response.choices[0].message.content == "sent" + + +@pytest.mark.asyncio +async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_async_path( + httpx_transport, +): + import respx + + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + response = await litellm.acompletion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + ) + + assert len(recorder.bodies) == 2 + assert recorder.bodies[0]["tools"][0]["strict"] is True + assert "strict" not in recorder.bodies[1]["tools"][0] + assert response.choices[0].message.content == "sent" + + +@pytest.mark.asyncio +async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_async_path( + httpx_transport, +): + import respx + + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)]) + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + with pytest.raises(litellm.BadRequestError): + await litellm.acompletion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + ) + + assert len(recorder.bodies) == 2 diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 6461245bb2c..ffa6bc601e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -2366,6 +2366,7 @@ async def test_update_team_team_member_budget_not_passed_to_db( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + explicitly_set_fields=frozenset(), ): # Remove team_member_budget from updated_kv as the real function does result_kv = updated_kv.copy() @@ -2738,6 +2739,138 @@ async def test_upsert_team_member_budget_table_no_existing_budget(): assert "team_member_budget_duration" not in result +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_clears_duration_kept_budget(mock_db_client): + """ + A request that keeps team_member_budget but explicitly nulls + team_member_budget_duration must clear the reset period and its reset time. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {"team_member_budget_id": "existing_budget_123"} + + mock_db_client.db.litellm_budgettable.update = AsyncMock( + side_effect=lambda where, data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv={ + "team_id": "test_team_id", + "team_member_budget": 100.0, + "team_member_budget_duration": None, + }, + team_member_budget=100.0, + team_member_budget_duration=None, + explicitly_set_fields={ + "team_member_budget", + "team_member_budget_duration", + }, + ) + + written = mock_db_client.db.litellm_budgettable.update.call_args.kwargs["data"] + assert written["max_budget"] == 100.0 + assert written["budget_duration"] is None + assert written["budget_reset_at"] is None + assert "rpm_limit" not in written + assert "tpm_limit" not in written + assert result["metadata"]["team_member_budget_id"] == "existing_budget_123" + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_explicit_null_duration_does_not_inherit_team_duration( + mock_db_client, +): + """ + A first-time member budget with an explicitly null duration must never + reset, even when the team itself has a reset period. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = "30d" + + mock_db_client.db.litellm_budgettable.create = AsyncMock( + side_effect=lambda data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=team_table, + new_team_data_json={"team_id": "test_team_id"}, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + team_member_budget_duration=None, + explicitly_set_fields={ + "team_member_budget", + "team_member_budget_duration", + }, + ) + + written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"] + assert written["max_budget"] == 100.0 + assert "budget_duration" not in written + assert "budget_reset_at" not in written + assert result["metadata"]["team_member_budget_id"] == written["budget_id"] + assert "team_member_budget" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_inherits_team_duration_when_duration_omitted( + mock_db_client, +): + """ + Omitting team_member_budget_duration keeps the existing inheritance of the + team's own reset period. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = "30d" + + mock_db_client.db.litellm_budgettable.create = AsyncMock( + side_effect=lambda data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=team_table, + new_team_data_json={"team_id": "test_team_id"}, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + explicitly_set_fields={"team_member_budget"}, + ) + + written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"] + assert written["budget_duration"] == "30d" + assert written["budget_reset_at"] is not None + assert result["metadata"]["team_member_budget_id"] == written["budget_id"] + + @pytest.mark.asyncio async def test_update_team_with_team_member_budget_duration( disable_audit_logging_for_mocked_team, @@ -2799,6 +2932,7 @@ async def test_update_team_with_team_member_budget_duration( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + explicitly_set_fields=frozenset(), ): result_kv = updated_kv.copy() result_kv.pop("team_member_budget", None) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py index 3e8e1e9dff8..67113ed9145 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py @@ -1,3 +1,4 @@ +import json import pytest from unittest.mock import MagicMock, AsyncMock, patch from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles @@ -246,3 +247,241 @@ async def test_patch_prompt_row_deleted_mid_update_returns_404(): exc_info.value.detail == "Prompt with ID test_prompt not found in environment development" ) + + +def test_is_ambiguous_keyed_prompt_data_shapes(): + from litellm.proxy.prompts.prompt_endpoints import is_ambiguous_keyed_prompt_data + + keyed_with_id = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + flat_with_id = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"content": "AHOY", "metadata": {}}, + ) + keyed_without_id = PromptLiteLLMParams( + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + no_prompt_data = PromptLiteLLMParams( + prompt_id="agent-prompt", prompt_integration="dotprompt" + ) + empty_prompt_data = PromptLiteLLMParams( + prompt_id="agent-prompt", prompt_integration="dotprompt", prompt_data={} + ) + + assert is_ambiguous_keyed_prompt_data(keyed_with_id) is True + assert is_ambiguous_keyed_prompt_data(flat_with_id) is False + assert is_ambiguous_keyed_prompt_data(keyed_without_id) is False + assert is_ambiguous_keyed_prompt_data(no_prompt_data) is False + assert is_ambiguous_keyed_prompt_data(empty_prompt_data) is False + + +@pytest.mark.asyncio +async def test_create_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + Prompt, + create_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = Prompt( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await create_prompt(request=request, user_api_key_dict=mock_user_auth) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +@pytest.mark.asyncio +async def test_patch_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + PatchPromptRequest, + patch_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = PatchPromptRequest( + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await patch_prompt( + prompt_id="agent-prompt", + request=request, + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +@pytest.mark.asyncio +async def test_patch_prompt_info_only_keeps_legacy_keyed_row_patchable(): + from litellm.proxy.prompts.prompt_endpoints import PatchPromptRequest, patch_prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + legacy_params = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + target_row = MagicMock() + target_row.id = "row-1" + target_row.version = 1 + updated_row = MagicMock() + updated_row.model_dump.return_value = { + "prompt_id": "agent-prompt", + "version": 1, + "environment": "production", + "created_by": None, + "litellm_params": legacy_params.model_dump_json(), + "prompt_info": PromptInfo(prompt_type="db", environment="production").model_dump_json(), + } + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[target_row] + ) + mock_prisma_client.db.litellm_prompttable.update = AsyncMock(return_value=updated_row) + + existing_prompt = PromptSpec( + prompt_id="agent-prompt.v1", + litellm_params=legacy_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( # test-quality-ok: keeps the registry reload from touching global callback state + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry, + ): + mock_registry.get_prompt_by_id.return_value = existing_prompt + + await patch_prompt( + prompt_id="agent-prompt", + request=PatchPromptRequest(prompt_info=PromptInfo(prompt_type="db", environment="production")), + user_api_key_dict=mock_user_auth, + ) + + update_kwargs = mock_prisma_client.db.litellm_prompttable.update.await_args.kwargs + assert update_kwargs["where"] == {"id": "row-1"} + assert json.loads(update_kwargs["data"]["prompt_info"])["environment"] == "production" + assert json.loads(update_kwargs["data"]["litellm_params"])["prompt_data"] == { + "json_prompt": {"content": "AHOY", "metadata": {}} + } + + +@pytest.mark.asyncio +async def test_update_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + Prompt, + update_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = Prompt( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await update_prompt( + prompt_id="agent-prompt", + request=request, + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +def test_create_versioned_prompt_spec_populates_version(): + from litellm.proxy.prompts.prompt_endpoints import create_versioned_prompt_spec + + db_prompt = MagicMock() + db_prompt.model_dump.return_value = { + "prompt_id": "agent-prompt", + "version": 3, + "environment": "development", + "created_by": "user-1", + "litellm_params": { + "prompt_id": "agent-prompt", + "prompt_integration": "dotprompt", + }, + "prompt_info": {"prompt_type": "db"}, + "created_at": None, + "updated_at": None, + } + + prompt_spec = create_versioned_prompt_spec(db_prompt=db_prompt) + + assert prompt_spec.prompt_id == "agent-prompt.v3" + assert prompt_spec.version == 3 + + +def test_initialize_prompt_keeps_version_and_created_by(): + import litellm + from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry + + registry = InMemoryPromptRegistry() + prompt_spec = PromptSpec( + prompt_id="agent-prompt.v3", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"content": "AHOY", "metadata": {}}, + ), + prompt_info=PromptInfo(prompt_type="db"), + version=3, + environment="development", + created_by="user-1", + ) + + with patch.object(litellm.logging_callback_manager, "add_litellm_callback"): # test-quality-ok: keeps initialize_prompt from registering a global callback that would leak across tests + initialized_prompt = registry.initialize_prompt(prompt=prompt_spec) + + assert initialized_prompt is not None + assert initialized_prompt.version == 3 + assert initialized_prompt.created_by == "user-1" + assert initialized_prompt.environment == "development" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index afc42e8db45..fc9d0a96163 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -4741,6 +4741,90 @@ async def test_add_router_settings_from_db_config_merge_logic(): assert combined_settings["nested_config"] == expected_nested +@pytest.mark.asyncio +async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks(): + """ + Regression test for DB router_settings rows carrying explicit empty lists + (e.g. {"fallbacks": []} written by the dashboard's delete-last-fallback flow): + empty lists are "no value" and must not clobber config.yaml fallbacks, + matching _deep_merge_dicts semantics. Non-empty DB lists still win. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_router = MagicMock() + mock_router.update_settings = MagicMock() + + config_data = { + "router_settings": { + "fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + "context_window_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + "content_policy_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + } + } + + mock_db_config = MagicMock() + mock_db_config.param_value = { + "fallbacks": [], + "context_window_fallbacks": [], + "content_policy_fallbacks": [{"gpt-oss-120b": ["other-model"]}], + "model_group_alias": {}, + "num_retries": 3, + } + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config) + + await proxy_config._add_router_settings_from_db_config( + config_data=config_data, + llm_router=mock_router, + prisma_client=mock_prisma_client, + ) + + combined_settings = mock_router.update_settings.call_args.kwargs + assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["other-model"]}] + assert combined_settings["num_retries"] == 3 + + +@pytest.mark.asyncio +async def test_add_router_settings_from_db_config_empty_db_list_still_clears_unconfigured_key(): + """ + An empty DB list only yields to config.yaml where the yaml configures that key. + When the yaml router_settings has no fallbacks, a DB {"fallbacks": []} (the + dashboard's delete-last-fallback write) must still reach the router so the + running pods drop the deleted fallback without a restart. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_router = MagicMock() + mock_router.update_settings = MagicMock() + + config_data = {"router_settings": {"num_retries": 1}} + + mock_db_config = MagicMock() + mock_db_config.param_value = {"fallbacks": [], "model_group_alias": {}} + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config) + + await proxy_config._add_router_settings_from_db_config( + config_data=config_data, + llm_router=mock_router, + prisma_client=mock_prisma_client, + ) + + combined_settings = mock_router.update_settings.call_args.kwargs + assert combined_settings["fallbacks"] == [] + assert combined_settings["num_retries"] == 1 + + @pytest.mark.asyncio async def test_add_router_settings_from_db_config_edge_cases(): """ @@ -11304,9 +11388,7 @@ class TestRouterModelNameOnStreamingChunks: with patch.object(ProxyLogging, "_fire_deferred_stream_logging"): return [ data - async for data in async_data_generator( - mock_response, MagicMock(spec=UserAPIKeyAuth), request_data - ) + async for data in async_data_generator(mock_response, MagicMock(spec=UserAPIKeyAuth), request_data) ] @staticmethod diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 82ab3b3bd08..dc2fbe3ed73 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3909,6 +3909,167 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_ assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9) +def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map): + """A router-facing model_name alias containing "/" whose leading segment is NOT a + registered provider must not be double-prefixed into a non-existent cost key. + + Regression test for #38069: alias "vertex/claude-opus-5" (real deployment + "vertex_ai/claude-opus-5") was re-prefixed into "vertex_ai/vertex/claude-opus-5", + silently pricing every streamed request at $0. + """ + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert selected == "vertex_ai/claude-opus-5" + + +def test_select_model_name_strips_duplicated_region_segment(_local_model_cost_map): + """A "region/model" alias whose leading segment repeats the request's region must + resolve to the region-priced cost key instead of keeping the region segment twice.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="us-east-1/anthropic.claude-v2:1", + ) + response._hidden_params = {"region_name": "us-east-1"} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="bedrock", + ) + + assert selected == "bedrock/us-east-1/anthropic.claude-v2:1" + + +def test_completion_cost_nonzero_for_slash_alias_model_name(_local_model_cost_map): + """End-to-end cost through a "/"-containing alias must price above zero (#38069).""" + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {"custom_llm_provider": "vertex_ai"} + response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50) + + cost = litellm.completion_cost( + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert cost == pytest.approx(100 * 5e-6 + 50 * 2.5e-5, rel=1e-9) + + +def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map): + """An alias that resolves to no known cost key keeps the legacy double-prefixed name.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="team/nonsense-model", + ) + response._hidden_params = {} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert selected == "vertex_ai/team/nonsense-model" + + +def test_completion_cost_keeps_custom_priced_slash_router_id(_local_model_cost_map): + """A custom-priced router id containing "/" keeps its custom pricing instead of being + rewritten to the built-in key its suffix happens to match.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + litellm.register_model( + model_cost={ + "vertex/claude-opus-5": { + "input_cost_per_token": 7e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "vertex_ai", + } + } + ) + + selected = _select_model_name_for_cost_calc( + model="vertex_ai/claude-opus-5", + completion_response=None, + custom_pricing=True, + custom_llm_provider="vertex_ai", + router_model_id="vertex/claude-opus-5", + ) + assert selected == "vertex_ai/vertex/claude-opus-5" + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {"custom_llm_provider": "vertex_ai"} + response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50) + + cost = litellm.completion_cost( + completion_response=response, + custom_llm_provider="vertex_ai", + custom_pricing=True, + router_model_id="vertex/claude-opus-5", + ) + assert cost == pytest.approx(100 * 7e-6 + 50 * 8e-6, rel=1e-9) + + @pytest.mark.parametrize( ("model", "expected_1hr_rate"), [("claude-3-haiku-20240307", 5e-07), ("claude-3-opus-20240229", 3e-05)], diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b283d777524..366e510a94d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4452,14 +4452,101 @@ def test_get_prompt_cache_min_tokens_resolves_per_model( assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens -def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None: - """The same model can carry a different minimum per platform, so the threshold must come from - the platform's own cost-map entry rather than being derived from the model family name.""" - assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512 - assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024 - assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens( - model="anthropic.claude-fable-5" - ) +def test_get_prompt_cache_min_tokens_uniform_for_fable_5_across_platforms(local_model_cost_map: None) -> None: + """Anthropic removed the Amazon Bedrock override for Claude Fable 5, so its 512-token minimum + now applies on every platform. The Bedrock entries carried the old 1024 and the re-export + entries carried nothing, so the router judged 512-1023-token prefixes uncacheable and skipped + prompt-cache-affinity routing for prompts the provider demonstrably caches (issue #35011).""" + wrong: Final = { + model: get_prompt_cache_min_tokens(model=model) + for model, info in litellm.model_cost.items() + if "fable-5" in model + and info.get("supports_prompt_caching") + and get_prompt_cache_min_tokens(model=model) != 512 + } + assert not wrong, f"every Claude Fable 5 entry must carry prompt_cache_min_tokens 512: {wrong}" + + +ANTHROPIC_REEXPORT_CACHE_MIN: Final = { + "azure_ai/claude-fable-5": 512, + "azure_ai/claude-haiku-4-5": 4096, + "azure_ai/claude-opus-4-1": 1024, + "azure_ai/claude-opus-4-5": 4096, + "azure_ai/claude-opus-4-6": 4096, + "azure_ai/claude-opus-4-7": 2048, + "azure_ai/claude-opus-4-8": 1024, + "azure_ai/claude-sonnet-4-5": 1024, + "azure_ai/claude-sonnet-4-6": 1024, + "azure_ai/claude-sonnet-5": 1024, + "databricks/databricks-claude-haiku-4-5": 4096, + "databricks/databricks-claude-opus-4": 1024, + "databricks/databricks-claude-opus-4-1": 1024, + "databricks/databricks-claude-opus-4-5": 4096, + "databricks/databricks-claude-opus-4-6": 4096, + "databricks/databricks-claude-sonnet-4": 1024, + "databricks/databricks-claude-sonnet-4-5": 1024, + "databricks/databricks-claude-sonnet-4-6": 1024, + "openrouter/anthropic/claude-haiku-4.5": 4096, + "openrouter/anthropic/claude-opus-4": 1024, + "openrouter/anthropic/claude-opus-4.1": 1024, + "openrouter/anthropic/claude-opus-4.5": 4096, + "openrouter/anthropic/claude-opus-4.6": 4096, + "openrouter/anthropic/claude-opus-4.7": 2048, + "openrouter/anthropic/claude-sonnet-4": 1024, + "openrouter/anthropic/claude-sonnet-4.5": 1024, + "openrouter/anthropic/claude-sonnet-4.6": 1024, + "replicate/anthropic/claude-4-sonnet": 1024, + "replicate/anthropic/claude-4.5-haiku": 4096, + "replicate/anthropic/claude-4.5-sonnet": 1024, + "snowflake/claude-4-opus": 1024, + "snowflake/claude-4-sonnet": 1024, + "snowflake/claude-haiku-4-5": 4096, + "snowflake/claude-sonnet-4-5": 1024, + "snowflake/claude-sonnet-4-6": 1024, + "vercel_ai_gateway/anthropic/claude-haiku-4.5": 4096, + "vercel_ai_gateway/anthropic/claude-opus-4": 1024, + "vercel_ai_gateway/anthropic/claude-opus-4.1": 1024, + "vercel_ai_gateway/anthropic/claude-opus-4.5": 4096, + "vercel_ai_gateway/anthropic/claude-opus-4.6": 4096, + "vercel_ai_gateway/anthropic/claude-sonnet-4": 1024, + "vercel_ai_gateway/anthropic/claude-sonnet-4.5": 1024, + "vertex_ai/claude-fable-5": 512, + "vertex_ai/claude-fable-5@default": 512, +} + + +def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local_model_cost_map: None) -> None: + """Regression for issue #35011: these re-export entries carried no prompt_cache_min_tokens, so + they silently inherited the 1024 default. That skipped cache-affinity routing for Fable 5's + 512-1023-token prefixes and reported 1024-4095-token prompts as cacheable on the 2048/4096 + models. The entry must be explicit so a default change can never re-break them, which is why + this asserts the cost-map value itself and not just the resolver's answer.""" + wrong: Final = { + model: (litellm.model_cost[model].get("prompt_cache_min_tokens"), get_prompt_cache_min_tokens(model=model)) + for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items() + if litellm.model_cost[model].get("prompt_cache_min_tokens") != expected + or get_prompt_cache_min_tokens(model=model) != expected + } + assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}" + + +def test_anthropic_reexport_cache_minimums_present_in_root_cost_map() -> None: + """The root map ships to the CDN independently of the bundled backup, so both must carry the + minimum or proxies reading one of them regress to the 1024 default.""" + root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") + with open(root_map_path) as f: + root_map: Final = json.load(f) + wrong: Final = { + model: root_map[model].get("prompt_cache_min_tokens") + for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items() + if root_map[model].get("prompt_cache_min_tokens") != expected + } + fable_5_wrong: Final = { + model: info.get("prompt_cache_min_tokens") + for model, info in root_map.items() + if "fable-5" in model and info.get("supports_prompt_caching") and info.get("prompt_cache_min_tokens") != 512 + } + assert not wrong and not fable_5_wrong, f"root cost map diverges: {wrong | fable_5_wrong}" GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 78e5b572492..71bce0b254c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -17,15 +17,14 @@ import { ReasoningEffort, TierModelParamsByTier, pruneTierModelParams, - resolveComplexityDefaultModel, setTierModelReasoningEffort, - tierOptions, } from "./complexity_router_tiers"; import TierModelEffortRows from "./TierModelEffortRows"; import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import { type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; @@ -37,12 +36,12 @@ export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; -export interface ComplexityTiers { +export type ComplexityTiers = { SIMPLE: string[]; MEDIUM: string[]; COMPLEX: string[]; REASONING: string[]; -} +}; export type ClassificationRubric = "legacy" | "agentic" | "chat" | "business"; @@ -224,10 +223,6 @@ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label; -/** Tiers the plan-mode floor may name: the backend rejects a floor whose tier has no models. */ -export const planModeEligibleTiers = (tiers: ComplexityTiers): Array => - TIER_KEYS.filter((tier) => (tiers[tier] ?? []).length > 0); - const ComplexityRouterConfig: React.FC = ({ modelInfo, value, @@ -246,12 +241,12 @@ const ComplexityRouterConfig: React.FC = ({ onEscalationKeywordsChange, showValidationErrors = false, }) => { - const planModeTiers = planModeEligibleTiers(value.tiers); - const planModeTierOptions = tierOptions(value.tier_labels).filter((option) => - (planModeTiers as string[]).includes(option.value), - ); - const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers); - const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model); + const tierRows = activeTierRows(value); + const planModeTierOptions = tierRows + .filter((row) => row.models.length > 0) + .map((row) => ({ value: row.id, label: effectiveTierLabel(row.id as keyof ComplexityTiers, value.tier_labels) })); + const derivedDefaultModel = resolveComplexityDefaultModel(value); + const defaultModel = resolveComplexityDefaultModel(value, value.default_model); // An absent list means the proxy does not send the field yet, so every level is offered as before. // An empty list is the group's own answer that its deployments share no level, and is left empty. @@ -325,12 +320,13 @@ const ComplexityRouterConfig: React.FC = ({ - {TIER_KEYS.map((tier, index) => { + {tierRows.map((row: TierRow, index) => { + const tier = row.id as keyof ComplexityTiers; const tierInfo = TIER_DESCRIPTIONS[tier]; const label = effectiveTierLabel(tier, value.tier_labels); - const tierMissing = showValidationErrors && value.tiers[tier].length === 0; + const tierMissing = showValidationErrors && row.models.length === 0; return ( -
+
{index > 0 && }
@@ -339,7 +335,7 @@ const ComplexityRouterConfig: React.FC = ({ - Tier {index + 1} of {TIER_KEYS.length} · {tier} + Tier {index + 1} of {tierRows.length} · {row.id}
Examples: {tierInfo.examples} @@ -364,7 +360,7 @@ const ComplexityRouterConfig: React.FC = ({ handleTierChange(tier, models)} placeholder={`Select model(s) for ${label.toLowerCase()} queries`} emptyText="No models found" @@ -372,12 +368,12 @@ const ComplexityRouterConfig: React.FC = ({ /> handleTierModelEffortChange(tier, model, effort)} /> - {value.tiers[tier].length > 1 && ( + {row.models.length > 1 && ( Multiple models selected — the router randomly picks among them per request (or Thompson-samples within the pool when adaptive routing is on). @@ -483,9 +479,12 @@ const ComplexityRouterConfig: React.FC = ({
- onChange({ ...value, plan_mode_min_tier: enabled ? planModeTiers.at(-1) : undefined }) + onChange({ + ...value, + plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined, + }) } aria-label="Route plan-mode requests to a minimum tier" /> @@ -494,7 +493,7 @@ const ComplexityRouterConfig: React.FC = ({ Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. The classifier still wins when it picks higher, and the override only lasts while plan mode is active. - {planModeTiers.length === 0 && " Add models to a tier to enable this."} + {planModeTierOptions.length === 0 && " Add models to a tier to enable this."} {value.plan_mode_min_tier !== undefined && (
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 98ee2b7ae7c..87c4754f56d 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -22,7 +22,6 @@ import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, - ComplexityTiers, DEFAULT_ADAPTIVE_WEIGHTS, DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, @@ -40,7 +39,9 @@ import { getSemanticConfigError, getTierLabelsError, } from "./build_complexity_router_config"; -import { resolveComplexityDefaultModel } from "./complexity_router_tiers"; +import { activeTierName, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; +import { DEFAULT_TIER_LABELS } from "./complexity_router_tiers"; +import type { ComplexityTier } from "./KeywordTierRules"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import AutoRouterRoutingTest from "./AutoRouterRoutingTest"; @@ -104,17 +105,10 @@ const presets = getAllPresets(); // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. -const tierConfigSummary = (tiers: ComplexityTiers): string => { - const parts = ( - [ - ["Simple", tiers.SIMPLE], - ["Medium", tiers.MEDIUM], - ["Complex", tiers.COMPLEX], - ["Reasoning", tiers.REASONING], - ] as const - ) - .filter(([, models]) => models.length > 0) - .map(([label, models]) => `${label}: ${models.join(", ")}`); +const tierConfigSummary = (config: ComplexityRouterConfigValue): string => { + const parts = activeTierRows(config) + .filter((row) => row.models.length > 0) + .map((row) => `${DEFAULT_TIER_LABELS[row.id as ComplexityTier] ?? activeTierName(row)}: ${row.models.join(", ")}`); return parts.length > 0 ? parts.join(" · ") : "No tiers configured yet"; }; @@ -128,9 +122,9 @@ const getSubmitBlockedReason = ( referencedModelsParams: Parameters[0], availability: ModelAvailability, ): string | null => - getMissingTiersError(config.tiers) ?? + getMissingTiersError(activeTierRows(config)) ?? getTierLabelsError(config.tier_labels) ?? - getPlanModeTierError(config.plan_mode_min_tier, config.tiers) ?? + getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ?? getKeywordTierRulesError(keywordTierRules) ?? getReferencedModelsError(referencedModelsParams, availability); @@ -378,7 +372,7 @@ const AddAutoRouterTab: React.FC = ({ const submitRecommendedRouter = async (name: string) => { const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams; - const missingTiersError = getMissingTiersError(tiers); + const missingTiersError = getMissingTiersError(activeTierRows(complexityRouterConfig)); if (missingTiersError) { setShowValidationErrors(true); toast.fromError(missingTiersError); @@ -423,7 +417,7 @@ const AddAutoRouterTab: React.FC = ({ return; } - const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model); + const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model); const validatedFields = requiresTeamScope ? (["auto_router_name", "team_id"] as const) : (["auto_router_name"] as const); @@ -463,10 +457,12 @@ const AddAutoRouterTab: React.FC = ({ const handleTestConnection = () => { const testTargetParams = { - tiers: complexityRouterConfig.tiers, + tiers: activeTierRows(complexityRouterConfig).map( + (row) => [activeTierName(row), row.models] as [string, string[]], + ), semanticMatchingEnabled, embeddingModel, - defaultModel: resolveComplexityDefaultModel(complexityRouterConfig.tiers, complexityRouterConfig.default_model), + defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model), }; const targets = buildAutoRouterTestTargets(testTargetParams); @@ -581,7 +577,7 @@ const AddAutoRouterTab: React.FC = ({ {!detailsExpanded && ( - {tierConfigSummary(complexityRouterConfig.tiers)} + {tierConfigSummary(complexityRouterConfig)} )} @@ -694,10 +690,7 @@ const AddAutoRouterTab: React.FC = ({ @@ -707,7 +700,6 @@ const AddAutoRouterTab: React.FC = ({ - , ] @@ -744,7 +736,6 @@ const AddAutoRouterTab: React.FC = ({ > Close - , ] diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts index 85fd846ddbc..7c29ea53060 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts @@ -1,11 +1,18 @@ import { buildAutoRouterTestTargets } from "./build_auto_router_test_targets"; -const tiers = { - SIMPLE: ["gpt-4o-mini"], - MEDIUM: ["claude-sonnet-4"], - COMPLEX: ["claude-sonnet-4"], - REASONING: ["o3"], -}; +const tierEntries = ( + SIMPLE: string[], + MEDIUM: string[] = [], + COMPLEX: string[] = [], + REASONING: string[] = [], +): [string, string[]][] => [ + ["SIMPLE", SIMPLE], + ["MEDIUM", MEDIUM], + ["COMPLEX", COMPLEX], + ["REASONING", REASONING], +]; + +const tiers = tierEntries(["gpt-4o-mini"], ["claude-sonnet-4"], ["claude-sonnet-4"], ["o3"]); describe("buildAutoRouterTestTargets", () => { it("dedups tiers that share a model group into one chat target carrying both labels", () => { @@ -19,7 +26,7 @@ describe("buildAutoRouterTestTargets", () => { it("emits a target per model when a tier has more than one, and dedups across tiers", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini", "claude-sonnet-4"], MEDIUM: ["claude-sonnet-4"], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini", "claude-sonnet-4"], ["claude-sonnet-4"]), semanticMatchingEnabled: false, embeddingModel: undefined, }); @@ -31,7 +38,7 @@ describe("buildAutoRouterTestTargets", () => { it("drops empty/whitespace tiers", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [" "], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"], [], [" "]), semanticMatchingEnabled: false, embeddingModel: undefined, }); @@ -41,7 +48,7 @@ describe("buildAutoRouterTestTargets", () => { it("returns [] when no tier is configured", () => { expect( buildAutoRouterTestTargets({ - tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries([]), semanticMatchingEnabled: false, embeddingModel: undefined, }), @@ -50,7 +57,7 @@ describe("buildAutoRouterTestTargets", () => { it("appends an embedding target only when semantic matching is on and a model is set", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", }); @@ -62,7 +69,7 @@ describe("buildAutoRouterTestTargets", () => { it("omits the embedding target when semantic matching is on but no model is chosen", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: true, embeddingModel: undefined, }); @@ -71,7 +78,7 @@ describe("buildAutoRouterTestTargets", () => { it("omits the embedding target when a model is set but semantic matching is off", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: false, embeddingModel: "voyage-3-5", }); @@ -112,7 +119,7 @@ describe("buildAutoRouterTestTargets", () => { it.each([[undefined], [""], [" "]])("adds no default target for %o", (defaultModel) => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: false, embeddingModel: undefined, defaultModel, diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts index 471552fc84f..70b92dbf8cc 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts @@ -1,5 +1,3 @@ -import { ComplexityTiers } from "./ComplexityRouterConfig"; - export type AutoRouterTestMode = "chat" | "embedding"; export interface AutoRouterTestTarget { @@ -9,7 +7,8 @@ export interface AutoRouterTestTarget { } export interface BuildAutoRouterTestTargetsParams { - tiers: ComplexityTiers; + /** Ordered [tier name, model groups] entries of the active tier set. */ + tiers: readonly (readonly [string, string[]])[]; semanticMatchingEnabled: boolean; embeddingModel: string | undefined; /** The resolved default model - see resolveComplexityDefaultModel. A live fallback destination, @@ -17,23 +16,14 @@ export interface BuildAutoRouterTestTargetsParams { defaultModel?: string; } -// Keys drive iteration order; `satisfies Record` makes it a -// compile error to add a tier to ComplexityTiers without listing it here (and vice versa). -const TIER_ORDER = Object.keys({ - SIMPLE: null, - MEDIUM: null, - COMPLEX: null, - REASONING: null, -} satisfies Record) as (keyof ComplexityTiers)[]; - export const buildAutoRouterTestTargets = ({ tiers, semanticMatchingEnabled, embeddingModel, defaultModel, }: BuildAutoRouterTestTargetsParams): AutoRouterTestTarget[] => { - const tieredByModel = TIER_ORDER.reduce>((acc, tier) => { - return (tiers[tier] ?? []).reduce((tierAcc, rawModel) => { + const tieredByModel = tiers.reduce>((acc, [tier, models]) => { + return models.reduce((tierAcc, rawModel) => { const modelGroup = rawModel?.trim(); if (!modelGroup) return tierAcc; return { ...tierAcc, [modelGroup]: [...(tierAcc[modelGroup] ?? []), tier] }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 33f0fb8a539..63545f5b7de 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -9,6 +9,7 @@ import { hydrateTierLabels, BuildComplexityRouterConfigParams, } from "./build_complexity_router_config"; +import { activeTierRows } from "./tier_rows"; const tiers = { SIMPLE: ["gpt-4o-mini"], @@ -275,30 +276,30 @@ describe("buildComplexityRouterConfig", () => { describe("getMissingTiersError", () => { it("returns null when all four tiers have a model", () => { - expect(getMissingTiersError(tiers)).toBeNull(); + expect(getMissingTiersError(activeTierRows({ tiers: tiers }))).toBeNull(); }); it("names the specific missing tier when only one is blank", () => { - expect(getMissingTiersError({ ...tiers, REASONING: [] })).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, REASONING: [] } }))).toBe( "Select a model for the following tier(s): REASONING", ); }); it("names multiple missing tiers in SIMPLE/MEDIUM/COMPLEX/REASONING order", () => { - expect(getMissingTiersError({ ...tiers, SIMPLE: [], REASONING: [] })).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: [], REASONING: [] } }))).toBe( "Select a model for the following tier(s): SIMPLE, REASONING", ); }); it("names all four tiers when none are filled", () => { const noTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }; - expect(getMissingTiersError(noTiers)).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: noTiers }))).toBe( "Select a model for the following tier(s): SIMPLE, MEDIUM, COMPLEX, REASONING", ); }); it("treats a tier with more than one model as filled", () => { - expect(getMissingTiersError({ ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] })).toBeNull(); + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] } }))).toBeNull(); }); }); @@ -645,15 +646,15 @@ describe("getPlanModeTierError", () => { const tiersWithEmptyComplex = { SIMPLE: ["m1"], MEDIUM: ["m1"], COMPLEX: [], REASONING: [] }; it("passes when the override is off", () => { - expect(getPlanModeTierError(undefined, tiersWithEmptyComplex)).toBeNull(); + expect(getPlanModeTierError(undefined, activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull(); }); it("passes when the named tier has models", () => { - expect(getPlanModeTierError("MEDIUM", tiersWithEmptyComplex)).toBeNull(); + expect(getPlanModeTierError("MEDIUM", activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull(); }); it("blocks a tier whose models were removed, which the backend would reject with a 400", () => { - expect(getPlanModeTierError("COMPLEX", tiersWithEmptyComplex)).toContain("COMPLEX"); + expect(getPlanModeTierError("COMPLEX", activeTierRows({ tiers: tiersWithEmptyComplex }))).toContain("COMPLEX"); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index bd95bea226a..241cae25705 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,4 +1,5 @@ import { KeywordTierRule } from "./KeywordTierRules"; +import { type TierRow, activeTierName, tierRowById } from "./tier_rows"; import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; import { TierModelParams, TierModelParamsByTier, serializeTierModelConfigs } from "./complexity_router_tiers"; import { @@ -10,6 +11,7 @@ import { ComplexityTierLabels, ComplexityTiers, DimensionWeights, + TIER_KEYS, TIER_DESCRIPTIONS, TierBoundaries, TokenThresholds, @@ -135,8 +137,6 @@ export interface ComplexityRouterConfigPayload { tier_model_configs?: Record; } -const TIER_KEYS: Array = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; - export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => { const renamed = TIER_KEYS.map((tier) => [tier, tierLabels?.[tier]?.trim() ?? ""] as const).filter( ([tier, label]) => label !== "" && label !== TIER_DESCRIPTIONS[tier].label, @@ -171,26 +171,20 @@ export const getTierLabelsError = (tierLabels: ComplexityTierLabels | undefined) return null; }; -// Requires all 4 tiers non-empty, so the create form can never reach the -// resolveComplexityDefaultModel(tiers, ...) === undefined case — MEDIUM (or SIMPLE) is always -// populated. The edit modal has no equivalent of this check (it allows saving with only some -// tiers filled), which is why it needs its own explicit `!defaultModel` guard after deriving — -// see edit_auto_router_modal.tsx's save handler. A future contributor copying this form's submit -// handler elsewhere should not assume the same guarantee holds without this check. -export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { - const missing = TIER_KEYS.filter((tier) => tiers[tier].length === 0); +// Requires every active tier non-empty, so the create form can never reach the +// resolveComplexityDefaultModel === undefined case. The edit modal allows a partially filled +// set, which is why it keeps its own !defaultModel guard after deriving. +export const getMissingTiersError = (rows: readonly TierRow[]): string | null => { + const missing = rows.filter((row) => row.models.length === 0).map(activeTierName); if (missing.length === 0) return null; return `Select a model for the following tier(s): ${missing.join(", ")}`; }; -// The backend rejects a plan-mode floor naming a tier with no models. The create form's -// getMissingTiersError makes this unreachable there; the edit modal allows partially filled -// tiers, so both gates call this to keep the two forms symmetric. -export const getPlanModeTierError = (planModeMinTier: string | undefined, tiers: ComplexityTiers): string | null => { +export const getPlanModeTierError = (planModeMinTier: string | undefined, rows: readonly TierRow[]): string | null => { if (!planModeMinTier) return null; - const models = tiers[planModeMinTier as keyof ComplexityTiers] ?? []; - if (models.length > 0) return null; - return `The plan-mode minimum tier (${planModeMinTier}) has no models. Add one or turn the override off.`; + const floor = tierRowById(rows, planModeMinTier); + if (floor && floor.models.length > 0) return null; + return `The plan-mode minimum tier (${floor ? activeTierName(floor) : planModeMinTier}) has no models. Add one or turn the override off.`; }; export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => { diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts index 4dffbbd2ac7..be48ff4958d 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts @@ -4,10 +4,10 @@ import { hydrateTierModelParams, normalizeTierModels, pruneTierModelParams, - resolveComplexityDefaultModel, serializeTierModelConfigs, setTierModelReasoningEffort, } from "./complexity_router_tiers"; +import { resolveComplexityDefaultModel } from "./tier_rows"; import type { ComplexityTiers } from "./ComplexityRouterConfig"; @@ -50,31 +50,31 @@ describe("resolveComplexityDefaultModel", () => { const noTiers: ComplexityTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }; it("derives from MEDIUM first when nothing is pinned", () => { - expect(resolveComplexityDefaultModel(tiers)).toBe("medium-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers })).toBe("medium-model"); }); it("falls back to SIMPLE when MEDIUM is empty", () => { - expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [] })).toBe("simple-model"); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("simple-model"); }); it("derives nothing from COMPLEX or REASONING, which the backend never falls through to", () => { - expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [], SIMPLE: [] })).toBeUndefined(); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [], SIMPLE: [] } })).toBeUndefined(); }); it("lets a pin beat the tiers rather than merely filling in for them", () => { - expect(resolveComplexityDefaultModel(tiers, "pinned-model")).toBe("pinned-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers }, "pinned-model")).toBe("pinned-model"); }); it("stands alone as the default when no tier holds a model", () => { - expect(resolveComplexityDefaultModel(noTiers, "pinned-model")).toBe("pinned-model"); + expect(resolveComplexityDefaultModel({ tiers: noTiers }, "pinned-model")).toBe("pinned-model"); }); it.each([[""], [" "], [undefined]])("reads %o as no pin and goes back to the tiers", (pinned) => { - expect(resolveComplexityDefaultModel(tiers, pinned)).toBe("medium-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers }, pinned)).toBe("medium-model"); }); it("resolves to nothing when neither a pin nor a tier offers a model", () => { - expect(resolveComplexityDefaultModel(noTiers)).toBeUndefined(); + expect(resolveComplexityDefaultModel({ tiers: noTiers })).toBeUndefined(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index 2ea1915ca03..ea0d34f6581 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -1,5 +1,5 @@ -import type { ComplexityTiers } from "./ComplexityRouterConfig"; import type { ComplexityTier } from "./KeywordTierRules"; +import { TIER_ORDER } from "./tier_rows"; export type TierModelParams = Record; @@ -80,13 +80,13 @@ export const hydrateTierModelParams = ( * tiers this editor does not render pass through rather than being dropped now the key is managed. */ export const serializeTierModelConfigs = ( - tiers: ComplexityTiers, + tiers: Record, tierModelParams: TierModelParamsByTier | undefined, ): Record | undefined => { if (tierModelParams === undefined) return undefined; const serialized = Object.entries(tierModelParams) .map(([tier, byModel]) => { - const selected = (TIER_ORDER as string[]).includes(tier) ? new Set(tiers[tier as ComplexityTier]) : undefined; + const selected = tier in tiers ? new Set(tiers[tier]) : undefined; const entries = Object.entries(byModel) .filter(([model, params]) => (selected === undefined || selected.has(model)) && Object.keys(params).length > 0) .map(([model_name, litellm_params]) => ({ model_name, litellm_params })); @@ -126,14 +126,6 @@ export const pruneTierModelParams = ( return Object.keys(next).length > 0 ? next : undefined; }; -/** - * Mirrors `init_complexity_router_deployment` (litellm/router.py): an explicit pin wins, otherwise - * the default is `MEDIUM or SIMPLE`. Deriving past SIMPLE would name a model the backend never - * picks, and it raises rather than falling through to COMPLEX/REASONING. - */ -export const resolveComplexityDefaultModel = (tiers: ComplexityTiers, pinned?: string): string | undefined => - pinned?.trim() || tiers.MEDIUM[0] || tiers.SIMPLE[0]; - export const DEFAULT_TIER_LABELS: Record = { SIMPLE: "Simple", MEDIUM: "Medium", @@ -141,8 +133,6 @@ export const DEFAULT_TIER_LABELS: Record = { REASONING: "Reasoning", }; -export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; - export const tierOptions = ( tierLabels: Partial> | undefined, ): { value: ComplexityTier; label: string }[] => diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts new file mode 100644 index 00000000000..54d9e4f3f0a --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { + activeTierName, + activeTierRows, + isBuiltInTierName, + resolveComplexityDefaultModel, + sameTierIdentity, + tierRowById, + tierRowByName, +} from "./tier_rows"; + +const tiers = { SIMPLE: ["a"], MEDIUM: ["b"], COMPLEX: ["c"], REASONING: ["d"] }; + +describe("activeTierRows", () => { + it("reads the tier set as rows whose id is the canonical tier key, in severity order", () => { + expect(activeTierRows({ tiers })).toEqual([ + { id: "SIMPLE", name: "SIMPLE", models: ["a"] }, + { id: "MEDIUM", name: "MEDIUM", models: ["b"] }, + { id: "COMPLEX", name: "COMPLEX", models: ["c"] }, + { id: "REASONING", name: "REASONING", models: ["d"] }, + ]); + }); + + it("gives a tier with no models an empty pool rather than dropping the row", () => { + expect(activeTierRows({ tiers: { ...tiers, COMPLEX: [] } })[2]).toEqual({ + id: "COMPLEX", + name: "COMPLEX", + models: [], + }); + }); + + it("finds a row by id and by name", () => { + const rows = activeTierRows({ tiers }); + expect(tierRowById(rows, "MEDIUM")?.models).toEqual(["b"]); + expect(tierRowById(rows, undefined)).toBeUndefined(); + expect(tierRowByName(rows, " medium ")?.id).toBe("MEDIUM"); + }); +}); + +describe("sameTierIdentity", () => { + it.each([ + ["AUDIT", "audit", true], + ["AUDIT", " audit ", true], + ["AUDIT", "AUDITS", false], + ])("compares %s and %s casefold, matching the backend's uniqueness rule", (left, right, expected) => { + expect(sameTierIdentity(left, right)).toBe(expected); + }); + + it("recognises the four built-in names regardless of case", () => { + expect(["SIMPLE", "medium", "Complex", "REASONING"].every(isBuiltInTierName)).toBe(true); + expect(isBuiltInTierName("SECURITY_REVIEW")).toBe(false); + }); + + it("trims a row name, since the backend matches fallback_tier and keyword rules exactly", () => { + expect(activeTierName({ id: "1", name: " AUDIT ", models: [] })).toBe("AUDIT"); + }); +}); + +describe("resolveComplexityDefaultModel", () => { + it("mirrors init_complexity_router_deployment: a pin wins, then MEDIUM, then SIMPLE", () => { + expect(resolveComplexityDefaultModel({ tiers }, "pinned")).toBe("pinned"); + expect(resolveComplexityDefaultModel({ tiers })).toBe("b"); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("a"); + }); + + it("resolves to nothing rather than falling through to COMPLEX, which the backend never picks", () => { + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, SIMPLE: [], MEDIUM: [] } })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts new file mode 100644 index 00000000000..c320980916a --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -0,0 +1,40 @@ +import type { ComplexityTiers } from "./ComplexityRouterConfig"; +import type { ComplexityTier } from "./KeywordTierRules"; + +export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; + +export interface TierRow { + id: string; + name: string; + models: string[]; +} + +export interface ActiveTierSet { + tiers: ComplexityTiers; +} + +export const activeTierName = (row: TierRow): string => row.name.trim(); + +export const sameTierIdentity = (left: string, right: string): boolean => + left.trim().toLowerCase() === right.trim().toLowerCase(); + +export const isBuiltInTierName = (name: string): boolean => TIER_ORDER.some((tier) => sameTierIdentity(tier, name)); + +// The only reader of the tier set. A row's id is the canonical tier key, so anything pointing into +// the set (the plan-mode floor, per-model params) points at a row rather than at a position. +export const activeTierRows = (value: ActiveTierSet): TierRow[] => + TIER_ORDER.map((tier) => ({ id: tier, name: tier, models: value.tiers[tier] ?? [] })); + +export const tierRowById = (rows: readonly TierRow[], id: string | undefined): TierRow | undefined => + id === undefined ? undefined : rows.find((row) => row.id === id); + +export const tierRowByName = (rows: readonly TierRow[], name: string): TierRow | undefined => + rows.find((row) => sameTierIdentity(row.name, name)); + +// Mirrors init_complexity_router_deployment (litellm/router.py): a pin wins, then MEDIUM or SIMPLE +// looked up by exact name. +export const resolveComplexityDefaultModel = (value: ActiveTierSet, pinned?: string): string | undefined => { + const rows = activeTierRows(value); + const named = (name: string) => rows.find((row) => activeTierName(row) === name)?.models[0]; + return pinned?.trim() || named("MEDIUM") || named("SIMPLE"); +}; diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx deleted file mode 100644 index 49a3ae8a001..00000000000 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import { useState } from "react"; -import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; -import { describe, it, expect, vi } from "vitest"; -import DurationSelect from "./DurationSelect"; - -describe("DurationSelect", () => { - it("should render", () => { - render(); - expect(screen.getByRole("combobox")).toBeInTheDocument(); - }); - - it("should render all three duration options", async () => { - const user = userEvent.setup(); - render(); - - const select = screen.getByRole("combobox"); - await user.click(select); - - expect(screen.getByText("Daily")).toBeInTheDocument(); - expect(screen.getByText("Weekly")).toBeInTheDocument(); - expect(screen.getByText("Monthly")).toBeInTheDocument(); - const dailyLabel = screen.getByText("Daily"); - const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; - await user.click(dailyOption); - }); - - it("should apply className prop", () => { - render(); - const select = screen.getByRole("combobox"); - expect(select.closest(".test-class")).toBeInTheDocument(); - }); - - it("should call onChange when an option is selected", async () => { - const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - const onChange = vi.fn(); - render(); - - const select = screen.getByRole("combobox"); - await user.click(select); - - const dailyLabel = screen.getByText("Daily"); - const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; - await user.click(dailyOption); - - expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object)); - }); - - it("should accept and pass value prop to Select", () => { - render(); - const select = screen.getByRole("combobox"); - expect(select).toBeInTheDocument(); - }); - - it.each([ - ["24h", "Daily"], - ["7d", "Weekly"], - ["30d", "Monthly"], - ])("shows the human label on the trigger for %s", (value, label) => { - render(); - - expect(screen.getByRole("combobox")).toHaveTextContent(label); - }); - - it("shows the human label on the trigger after the user picks an option", async () => { - const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - const Harness = () => { - const [value, setValue] = useState("24h"); - return ; - }; - render(); - - await user.click(screen.getByRole("combobox")); - const monthly = screen.getByText("Monthly"); - await user.click(monthly.closest('[role="option"]') ?? monthly); - - expect(screen.getByRole("combobox")).toHaveTextContent("Monthly"); - }); -}); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx deleted file mode 100644 index 55e8aafbae7..00000000000 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; - -interface DurationSelectProps { - className?: string; - value?: string; - onChange?: (value: string, option: { value: string; label: string }) => void; -} - -const DURATION_OPTIONS = [ - { value: "24h", label: "Daily" }, - { value: "7d", label: "Weekly" }, - { value: "30d", label: "Monthly" }, -]; - -export default function DurationSelect({ className, value, onChange }: DurationSelectProps) { - return ( - - ); -} diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index bce96ec76f5..8f4b06d80fb 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -14,25 +14,21 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; -import { - hydrateTierModelParams, - normalizeTierModels, - resolveComplexityDefaultModel, - serializeTierModelConfigs, -} from "../add_model/complexity_router_tiers"; +import { hydrateTierModelParams, normalizeTierModels } from "../add_model/complexity_router_tiers"; +import { type ActiveTierSet, activeTierRows, resolveComplexityDefaultModel } from "../add_model/tier_rows"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; import { + type BuildComplexityRouterConfigParams, + buildComplexityRouterConfig, getKeywordTierRulesError, getSemanticConfigError, getPlanModeTierError, getTierLabelsError, hydrateTierLabels, - normalizeClassifierLlmConfig, - serializeTierLabels, } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching"; -import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model/complexity_router_keywords"; +import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords"; import { hydrateDimensionWeights, hydrateReasoningOverrideMinScore, @@ -46,7 +42,6 @@ import ComplexityRouterConfig, { DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, - heuristicScoringRole, } from "../add_model/ComplexityRouterConfig"; import { Dialog, @@ -119,12 +114,12 @@ const toRecord = (value: unknown): Record => { export const hydratePinnedDefaultModel = ( storedConfigDefaultModel: unknown, litellmParamsDefaultModel: string | null | undefined, - tiers: ComplexityTiers, + activeTiers: ActiveTierSet, ): string | undefined => { if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) { return storedConfigDefaultModel; } - const tierDerived = resolveComplexityDefaultModel(tiers); + const tierDerived = resolveComplexityDefaultModel(activeTiers); const externalOverride = litellmParamsDefaultModel?.trim(); return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined; }; @@ -148,73 +143,48 @@ export const buildUpdatedComplexityRouterConfig = ( if (keywordMatching !== undefined && KEYWORD_MATCHING_KEYS.has(key)) return true; return customTechnicalKeywords !== undefined && key === "custom_technical_keywords"; }; - const preservedConfig = Object.fromEntries(Object.entries(toRecord(storedConfig)).filter(([key]) => !isManaged(key))); - const adaptiveEligible = value.adaptive_eligible ?? "all"; - const storedKeywordRules = keywordMatching ? serializeKeywordTierRules(keywordMatching.keywordTierRules) : []; - const serializedTierLabels = serializeTierLabels(value.tier_labels); - const scorerRuns = heuristicScoringRole(value) !== "never"; - const serializedTierModelConfigs = serializeTierModelConfigs(value.tiers, value.tier_model_params); + const builderParams: BuildComplexityRouterConfigParams = { + tiers: value.tiers, + defaultModel: value.default_model, + planModeMinTier: value.plan_mode_min_tier, + tierLabels: value.tier_labels, + classifierType: value.classifier_type, + classifierLlmConfig: value.classifier_llm_config, + classifierContextWindowSize: value.classifier_context_window_size, + classifierContextBudgetChars: value.classifier_context_budget_chars, + classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, + classifierFallback: value.classifier_fallback, + sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, + deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + customTechnicalKeywords: customTechnicalKeywords ?? [], + keywordTierRules: keywordMatching?.keywordTierRules ?? [], + semanticMatchingEnabled: keywordMatching?.semanticMatchingEnabled ?? false, + embeddingModel: keywordMatching?.embeddingModel, + matchThreshold: keywordMatching?.matchThreshold ?? DEFAULT_MATCH_THRESHOLD, + escalationKeywords: keywordMatching?.escalationKeywords ?? [], + adaptive: value.adaptive ?? false, + adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, + tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, + adaptiveEligible: value.adaptive_eligible ?? "all", + returnRawModelName: value.return_raw_model_name ?? false, + tierBoundaries: value.tier_boundaries, + tokenThresholds: value.token_thresholds, + dimensionWeights: value.dimension_weights, + reasoningOverrideMinScore: value.reasoning_override_min_score, + tierModelParams: value.tier_model_params, + }; + const built = buildComplexityRouterConfig(builderParams); + // Keys this call does not own stay as the stored config left them. + const unowned: readonly string[] = [ + ...(keywordMatching === undefined ? KEYWORD_MATCHING_KEYS : []), + ...(customTechnicalKeywords === undefined ? ["custom_technical_keywords"] : []), + ]; return { ...preservedConfig, - tiers: value.tiers, - ...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }), - ...(value.default_model?.trim() && { default_model: value.default_model }), - ...(value.plan_mode_min_tier?.trim() && { plan_mode_min_tier: value.plan_mode_min_tier }), - ...(serializedTierLabels && { tier_labels: serializedTierLabels }), - classifier_type: value.classifier_type, - ...(value.classifier_type === "llm" && value.classifier_llm_config - ? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) } - : {}), - ...(value.classifier_type === "llm" && - value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }), - ...(value.classifier_type === "llm" && - value.classifier_context_window_size !== undefined && { - classifier_context_window_size: value.classifier_context_window_size, - }), - ...(value.classifier_type === "llm" && - value.classifier_context_budget_chars !== undefined && { - classifier_context_budget_chars: value.classifier_context_budget_chars, - }), - ...(value.classifier_type === "llm" && - value.classifier_context_include_assistant_turns !== undefined && { - classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns, - }), - session_affinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, - deployment_affinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, - ...(customTechnicalKeywords && - customTechnicalKeywords.length > 0 && { - custom_technical_keywords: customTechnicalKeywords, - }), - ...(value.adaptive && { - adaptive: true, - adaptive_weights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, - ...(adaptiveEligible === "all" && { - tier_distance_penalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, - }), - adaptive_eligible: adaptiveEligible, - }), - ...(value.return_raw_model_name && { return_raw_model_name: true }), - ...(keywordMatching && { - // Mirrors buildComplexityRouterConfig: the key only when there is a rule to write, - // escalation keywords always, semantic trio only when on. - ...(storedKeywordRules.length > 0 && { keyword_tier_rules: storedKeywordRules }), - escalation_keywords: keywordMatching.escalationKeywords.map((k) => k.trim()).filter(Boolean), - ...(keywordMatching.semanticMatchingEnabled && { - semantic_keyword_matching: true, - embedding_model: keywordMatching.embeddingModel, - match_threshold: keywordMatching.matchThreshold, - }), - }), - ...(scorerRuns && value.tier_boundaries !== undefined && { tier_boundaries: value.tier_boundaries }), - ...(scorerRuns && value.token_thresholds !== undefined && { token_thresholds: value.token_thresholds }), - ...(scorerRuns && value.dimension_weights !== undefined && { dimension_weights: value.dimension_weights }), - ...(scorerRuns && - value.reasoning_override_min_score !== undefined && { - reasoning_override_min_score: value.reasoning_override_min_score, - }), + ...Object.fromEntries(Object.entries(built).filter(([key]) => !unowned.includes(key))), }; }; @@ -297,7 +267,7 @@ const EditAutoRouterModal: React.FC = ({ ? "Please select at least one model for a complexity tier" : null) ?? getTierLabelsError(complexityRouterConfig.tier_labels) ?? - getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, complexityRouterConfig.tiers) ?? + getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ?? getKeywordTierRulesError(keywordTierRules); useEffect(() => { @@ -355,7 +325,7 @@ const EditAutoRouterModal: React.FC = ({ default_model: hydratePinnedDefaultModel( parsedConfig.default_model, modelData.litellm_params?.complexity_router_default_model, - hydratedTiers, + { tiers: hydratedTiers }, ), plan_mode_min_tier: typeof parsedConfig.plan_mode_min_tier === "string" && parsedConfig.plan_mode_min_tier.trim() !== "" @@ -486,7 +456,7 @@ const EditAutoRouterModal: React.FC = ({ // build_complexity_router_config.ts for why create never can). init_complexity_router_deployment // raises in that case (litellm/router.py), so block it rather than saving a router that // fails at init. - const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model); + const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model); if (!defaultModel) { setShowValidationErrors(true); toast.fromError( diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 65b0a18f78a..f21e98e084c 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -16,7 +16,7 @@ import { stripMaskedSecrets } from "../utils/maskedSecretUtils"; import { truncateString } from "../utils/textUtils"; import AutoRouterConnectionTest from "./add_model/auto_router_connection_test"; import { AutoRouterTestTarget, buildAutoRouterTestTargets } from "./add_model/build_auto_router_test_targets"; -import { normalizeTierModels, resolveComplexityDefaultModel } from "./add_model/complexity_router_tiers"; +import { normalizeTierModels } from "./add_model/complexity_router_tiers"; import { hasAutoRouterEditor, isAutoRouterDeployment, @@ -91,12 +91,10 @@ const buildComplexityRouterTestTargets = ( config = rawConfig; } - const tiers = { - SIMPLE: normalizeTierModels(config.tiers?.SIMPLE), - MEDIUM: normalizeTierModels(config.tiers?.MEDIUM), - COMPLEX: normalizeTierModels(config.tiers?.COMPLEX), - REASONING: normalizeTierModels(config.tiers?.REASONING), - }; + const tiers: [string, string[]][] = + config.tiers && typeof config.tiers === "object" + ? Object.entries(config.tiers).map(([tier, models]) => [tier, normalizeTierModels(models)]) + : []; // Mirrors init_complexity_router_deployment (litellm/router.py): litellm_params wins, otherwise // pure tier-derivation. complexity_router_config.default_model is a UI-only marker the backend @@ -108,7 +106,7 @@ const buildComplexityRouterTestTargets = ( tiers, semanticMatchingEnabled: Boolean(config.semantic_keyword_matching), embeddingModel: config.embedding_model, - defaultModel: resolveComplexityDefaultModel(tiers, effectiveDefaultModel), + defaultModel: effectiveDefaultModel, }; return buildAutoRouterTestTargets(testTargetParams); }; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index c3cc362e29e..d1978ca751f 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -4,7 +4,7 @@ import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils"; -import TeamInfoView from "./TeamInfo"; +import TeamInfoView, { type TeamData } from "./TeamInfo"; const authState = vi.hoisted(() => ({ userRole: "Admin" })); @@ -1613,10 +1613,18 @@ describe("TeamInfoView - which team member fields reach the update payload depen vi.clearAllMocks(); }); - const openEditor = async (user: ReturnType) => { + const openEditor = async ( + user: ReturnType, + teamMemberBudgetTable: TeamData["team_info"]["team_member_budget_table"] = { + max_budget: 42, + budget_duration: "30d", + tpm_limit: 11, + rpm_limit: 22, + }, + ) => { vi.mocked(networking.teamInfoCall).mockResolvedValue( createMockTeamData({ - team_member_budget_table: { max_budget: 42, budget_duration: "30d", tpm_limit: 11, rpm_limit: 22 }, + team_member_budget_table: teamMemberBudgetTable, default_team_member_models: ["gpt-4"], }), ); @@ -1667,6 +1675,46 @@ describe("TeamInfoView - which team member fields reach the update payload depen expect(payload.default_team_member_models).toEqual(["gpt-4"]); }); + it("sends a null team_member_budget_duration when Default Budget Duration is set to never reset", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user); + + await user.click(screen.getByText("Team Member Settings")); + await screen.findByLabelText("Default Budget (USD)"); + await chooseSelectOption(user, screen.getByLabelText("Default Budget Duration"), "Never resets"); + + const payload = await save(user); + + expect(payload.team_member_budget_duration).toBeNull(); + expect(payload.team_member_budget).toBe(42); + expect(JSON.stringify(payload)).toContain('"team_member_budget_duration":null'); + }); + + it("shows Never resets for a stored member budget whose duration is null", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user, { max_budget: 42, budget_duration: null, tpm_limit: null, rpm_limit: null }); + + await user.click(screen.getByText("Team Member Settings")); + + expect(await screen.findByLabelText("Default Budget Duration")).toHaveTextContent("Never resets"); + }); + + it("omits team_member_budget_duration when the dropdown is left untouched on a team with no member budget", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user, null); + + await user.click(screen.getByText("Team Member Settings")); + const durationSelect = await screen.findByLabelText("Default Budget Duration"); + expect(durationSelect).toHaveTextContent("Inherit team reset period"); + expect(durationSelect).not.toHaveTextContent("Never resets"); + await user.type(screen.getByLabelText("Default Budget (USD)"), "100"); + + const payload = await save(user); + + expect(payload.team_member_budget).toBe(100); + expect(JSON.parse(JSON.stringify(payload))).not.toHaveProperty("team_member_budget_duration"); + }); + it("omits object_permission.search_tools while Search Tool Settings is closed", async () => { const user = userEvent.setup({ delay: null }); await openEditor(user); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 0b2a2deb72d..6be1054c7fe 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -48,7 +48,7 @@ import { z } from "zod/v4"; import GuardrailsSelect from "./GuardrailsSelect"; import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; -import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; +import BudgetDurationDropdown, { NEVER_RESETS_BUDGET_DURATION } from "../common_components/budget_duration_dropdown"; import { computeTeamModelBadges, normalizeTeamModelSelection, @@ -64,7 +64,6 @@ import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMeta import ModelAliasManager from "../common_components/ModelAliasManager"; import AgentSelector from "../agent_management/AgentSelector"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; -import DurationSelect from "../common_components/DurationSelect"; import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; import GuardrailSettingsView from "../GuardrailSettingsView"; @@ -169,7 +168,7 @@ export interface TeamData { object_permission?: ObjectPermission | null; team_member_budget_table: { max_budget: number; - budget_duration: string; + budget_duration: string | null; tpm_limit: number | null; rpm_limit: number | null; } | null; @@ -1256,7 +1255,15 @@ const TeamInfoView: React.FC = ({ name="team_member_budget_duration" label="Default Budget Duration" > - {({ value, onChange }) => } + {({ id, value, onChange }) => ( + onChange(next === NEVER_RESETS_BUDGET_DURATION ? null : next)} + /> + )} , ): Set => { const { tiers, classifier_llm_config: classifier, embedding_model: embedding, default_model: pinned } = config; - const models = [ - ...tiers.SIMPLE, - ...tiers.MEDIUM, - ...tiers.COMPLEX, - ...tiers.REASONING, - classifier?.model, - embedding, - pinned, - ]; + const models = [...Object.values(tiers).flat(), classifier?.model, embedding, pinned]; // Boolean(), not != null: an empty-string placeholder (e.g. classifier_llm_config seeded before a // model is chosen) is never a real model reference either. return new Set(models.filter((model): model is string => Boolean(model))); @@ -191,7 +182,7 @@ export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability: // effect would block submit for a model that was never going to be submitted. export const getReferencedModelsError = ( params: { - tiers: ComplexityTiers; + tiers: ComplexityRouterConfigPayload["tiers"]; classifierType: ClassifierType; classifierLlmConfig: ClassifierLLMConfig | undefined; semanticMatchingEnabled: boolean; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bb6bcbe8752..d660d6b5aca 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11029,11 +11029,9 @@ export interface paths { * -d '{ * "prompt_id": "my_prompt", * "litellm_params": { - * "prompt_id": "json_prompt", + * "prompt_id": "my_prompt", * "prompt_integration": "dotprompt", - * ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED - * "prompt_directory": "/path/to/dotprompt/folder", - * "prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}} + * "prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}} * }, * "prompt_info": { * "prompt_type": "config"