diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6536941a094..4cd292b2416 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -794,14 +794,27 @@ def _select_model_name_for_cost_calc( and custom_llm_provider is not None and not _model_contains_known_llm_provider(return_model) ): # add provider prefix if not already present, to match model_cost - if region_name is not None: - return_model = f"{custom_llm_provider}/{region_name}/{return_model}" - else: - return_model = f"{custom_llm_provider}/{return_model}" + provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}" + return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name) return return_model +def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: + """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the + registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already + resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069).""" + segments: Final = model.split("/") + if "/".join(segments[1:]) in litellm.model_cost: + return model + head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1 + head: Final = "/".join(segments[:head_len]) + tail: Final = segments[head_len:] + strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail)) + candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1)) + return next((candidate for candidate in candidates if candidate in litellm.model_cost), model) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _model_contains_known_llm_provider(model: str) -> bool: """ diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 07d83bc34d5..1188bce27da 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom if dotprompt_content and not prompt_data and not prompt_file: prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content) + from .prompt_manager import strip_version_suffix + + registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id + try: dot_prompt_manager: Final = DotpromptManager( prompt_directory=prompt_directory, prompt_data=prompt_data, prompt_file=prompt_file, - prompt_id=prompt_id, + prompt_id=registration_prompt_id, ) return dot_prompt_manager diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index e5e868f0523..0f45e926c30 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement): if prompt_id is None: return False try: - return prompt_id in self.prompt_manager.list_prompts() + return self.prompt_manager.get_prompt(prompt_id) is not None except Exception: # If there's any error accessing prompts, don't run prompt management return False diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 46750ed9799..a0d5be71392 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment +def strip_version_suffix(prompt_id: str) -> str | None: + base, separator, version = prompt_id.rpartition(".v") + if separator and base and version.isdigit(): + return base + return None + + class PromptTemplate: """Represents a single prompt template with metadata and content.""" @@ -124,11 +131,13 @@ class PromptManager: "content": "template content", "metadata": {"model": "gpt-4", "temperature": 0.7, ...} } + prompt_id - """ - if prompt_id: - prompt_data = {prompt_id: prompt_data} - for prompt_id, prompt_info in prompt_data.items(): + A dict carrying a "content" key is a single flat template registered under + prompt_id; anything else is treated as already keyed by template ID. + """ + keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data + + for template_id, prompt_info in keyed_prompts.items(): try: content = prompt_info.get("content", "") metadata = prompt_info.get("metadata", {}) @@ -136,11 +145,11 @@ class PromptManager: template = PromptTemplate( content=content, metadata=metadata, - template_id=prompt_id, + template_id=template_id, ) - self.prompts[prompt_id] = template + self.prompts[template_id] = template except Exception: - # Optional: print(f"Error loading prompt from JSON: {prompt_id}") + # Optional: print(f"Error loading prompt from JSON: {template_id}") pass def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate: @@ -272,8 +281,12 @@ class PromptManager: if versioned_id in self.prompts: return self.prompts[versioned_id] - # Fall back to base prompt_id - return self.prompts.get(prompt_id) + direct_match: Final = self.prompts.get(prompt_id) + if direct_match is not None: + return direct_match + + base_prompt_id: Final = strip_version_suffix(prompt_id) + return self.prompts.get(base_prompt_id) if base_prompt_id else None def list_prompts(self) -> list[str]: """Get a list of all available prompt IDs.""" diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 43b2eca2fd5..c745bbea5c4 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,19 +3,25 @@ Helper functions for health check calls. """ import base64 -from collections.abc import Callable +from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Final, Literal from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import ImageResponse # Minimal PDF for health checks - base64 encoded 1-page PDF with just "test" TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y=" -# Minimal image for health checks - base64 encoded 512x512 solid-gray PNG -TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAFlklEQVR42u3VMQEAAAzCMKQjHQ97l0jo0xSAlyIBgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAUgAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAADcDrctaAb6XeXAAAAAASUVORK5CYII=" +# Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG +TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC" + + +IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = ( + "Add a small yellow star in the top right corner of this simple drawing of a blue circle on a white background" +) def get_image_file_for_health_check() -> bytes: @@ -121,6 +127,17 @@ class HealthCheckHelpers: else: return await litellm.acompletion(**model_params) + @staticmethod + async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse": + import litellm + + try: + return await edit_request() + except litellm.BadRequestError as e: + if isinstance(e, litellm.ContentPolicyViolationError) or "moderation_blocked" in str(e): + return litellm.ImageResponse() + raise + @staticmethod def get_mode_handlers( model: str, @@ -195,10 +212,12 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), prompt=prompt, ), - "image_edit": lambda: litellm.aimage_edit( - **_filter_model_params(model_params=model_params), - image=get_image_file_for_health_check(), - prompt=prompt or "test", + "image_edit": lambda: HealthCheckHelpers._image_edit_health_check( + edit_request=lambda: litellm.aimage_edit( + **_filter_model_params(model_params=model_params), + image=get_image_file_for_health_check(), + prompt=IMAGE_EDIT_HEALTH_CHECK_PROMPT, + ), ), "video_generation": lambda: litellm.avideo_generation( **_filter_model_params(model_params=model_params), diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 566c960333a..ea19a7c7ddf 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -2,16 +2,17 @@ Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions` """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping, Sequence from typing import Any, Final, Literal, cast, overload import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( - handle_messages_with_content_list_to_str_conversion, + convert_content_list_to_str, + extract_search_results_text, ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from litellm.utils import supports_reasoning +from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -117,13 +118,98 @@ class DeepSeekChatConfig(OpenAIGPTConfig): self, messages: list[AllMessageValues], model: str, is_async: bool = False ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: """ - DeepSeek does not support content in list format. + DeepSeek vision models accept image_url content blocks in user + messages (https://api-docs.deepseek.com/guides/vision), so those + content lists are forwarded as-is, with any search_results text + appended as a trailing text block. Every other message keeps the + historical string collapse (which also folds search_results text + into string content); a list with no extractable text stays + unchanged, matching what DeepSeek historically received. """ - messages = handle_messages_with_content_list_to_str_conversion(messages) + forward_images: Final = any( + isinstance(message.get("content"), list) for message in messages + ) and supports_vision(model=model, custom_llm_provider="deepseek") + transformed: Final = [ # mutable-ok: provider messages must stay JSON-array lists the base transform mutates + self._forward_or_collapse_content(message=message, forward_images=forward_images) for message in messages + ] + if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) + return super()._transform_messages(messages=transformed, model=model, is_async=True) else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed, model=model, is_async=False) + + def _forward_or_collapse_content(self, message: AllMessageValues, forward_images: bool) -> AllMessageValues: + """ + Returns the vision-forwardable message with any search_results text + appended as a text block; every other message keeps the historical + string collapse, which extracts the text from a content list and + folds search_results text into string content. + """ + content: Final = message.get("content") + if ( + forward_images + and isinstance(content, list) + and self._is_vision_forwardable_content(message=message, content=content) + ): + return self._with_search_results_text_block(message=message, content=content) + collapsed: Final = convert_content_list_to_str(message=message) + if not collapsed or collapsed == content: + return message + collapsed_message: Final = {**message, "content": collapsed} # mutable-ok: wire messages are plain JSON dicts + return cast(AllMessageValues, collapsed_message) # cast-ok: TypedDict spread narrows to dict + + def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool: + """ + True only for a user message whose content list holds well-formed + text and image_url blocks with at least one image; a block missing + its payload falls back to the string collapse instead of crashing + or reaching the wire malformed. The model capability gate lives in + the caller. + """ + if message.get("role") != "user": + return False + if not all(self._is_forwardable_block(block) for block in content): + return False + return any(isinstance(block, dict) and block.get("type") == "image_url" for block in content) + + @staticmethod + def _is_forwardable_block(block: object) -> bool: + """A dict block typed text or image_url that carries its payload.""" + if not isinstance(block, dict): + return False + block_type: Final = block.get("type") + if block_type == "image_url": + return DeepSeekChatConfig._is_image_url_payload(block.get("image_url")) + if block_type == "text": + return isinstance(block.get("text"), str) + return False + + @staticmethod + def _is_image_url_payload(payload: object) -> bool: + """A url string or an object carrying one, per the OpenAI image_url shape.""" + if isinstance(payload, str): + return bool(payload) + if not isinstance(payload, Mapping): + return False + url: Final = payload.get("url") + return isinstance(url, str) and bool(url) + + def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues: + """ + Appends the message's search_results text as a trailing text block, + keeping the context that the string collapse used to fold in, and + drops the non-OpenAI search_results key from the wire message. + """ + message_fields: Final = cast(Mapping[str, object], message) # cast-ok: search_results is not on the TypedDicts + search_text: Final = extract_search_results_text(message_fields.get("search_results")) + if not search_text: + return message + forwarded_content: Final = [*content, {"type": "text", "text": search_text}] # mutable-ok: JSON-array content + forwarded: Final = { # mutable-ok: wire messages are plain JSON dicts + **{key: value for key, value in message_fields.items() if key != "search_results"}, + "content": forwarded_content, + } + return cast(AllMessageValues, forwarded) # cast-ok: TypedDict spread narrows to dict def _thinking_mode_active(self, model: str, optional_params: dict) -> bool: """ diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 8e35cfebc5b..8c306faa036 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -13,6 +13,13 @@ class FireworksAIException(BaseLLMException): def get_fireworks_session_id(litellm_params: dict) -> str | None: + """ + Session id to send as `x-session-affinity`, or None when the caller gave none. + + Deliberately does not fall back to `litellm_trace_id`: that is generated per + request (`str(uuid.uuid4())` when absent), so using it pins every request to a + different Fireworks node and prompt caching never hits. + """ params: Final = litellm_params for key in ("litellm_session_id", "session_id"): value = params.get(key) @@ -23,9 +30,6 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None: value = metadata.get("session_id") if value: return str(value) - value = params.get("litellm_trace_id") - if value: - return str(value) return None diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index ea576750cf3..51801e91356 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -4,6 +4,7 @@ This file contains the transformation logic for the Gemini realtime API. import json from collections import OrderedDict +from collections.abc import Mapping from typing import Any, Final, cast import litellm @@ -72,6 +73,28 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT} +OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( + {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"} +) + + +def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None: + """Build the Gemini Live speechConfig for a client-requested voice. + + OpenAI stock voice names have no Gemini equivalent and Gemini Live closes + the session on an unknown voice, so they are dropped with a warning and + the model keeps its default voice. Every other name is forwarded verbatim. + """ + if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES: + verbose_logger.warning( + "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; " + "dropping it so the session keeps the model's default voice.", + voice, + ) + return None + return VertexGeminiConfig()._map_audio_params({"voice": voice}) + + class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping @@ -282,12 +305,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): automaticActivityDetection=transformed_audio_activity_config ) elif key == "voice": - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - - vertex_gemini_config = VertexGeminiConfig() - speech_config = vertex_gemini_config._map_audio_params({"voice": value}) + speech_config = _gemini_live_speech_config(value) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: @@ -365,10 +383,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): entry: Final = GeminiRealtimeConfig._model_cost_entry(model) return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live")) - @staticmethod - def _is_native_audio_model(model: str) -> bool: - return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio")) - @staticmethod def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]: """Map unsupported TEXT responseModalities to AUDIO for audio-only Live models.""" @@ -384,7 +398,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): @staticmethod def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]: - """Drop fields Gemini Live native-audio rejects on ``setup``.""" generation_config: Final = setup.get("generationConfig") if isinstance(generation_config, dict): modalities: Final = generation_config.get("responseModalities") @@ -392,8 +405,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities( model, modalities ) - if GeminiRealtimeConfig._is_native_audio_model(model): - generation_config.pop("speechConfig", None) return setup def _handle_session_update( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d12ba24eda4..ccc86ef9ece 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1979,16 +1979,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _calculate_web_search_requests(grounding_metadata: list[dict]) -> int | None: - web_search_requests: int | None = None - - if grounding_metadata and isinstance(grounding_metadata, list) and len(grounding_metadata) > 0: - for grounding_metadata_item in grounding_metadata: - web_search_queries = grounding_metadata_item.get("webSearchQueries") - if web_search_queries and web_search_requests: - web_search_requests += len([q for q in web_search_queries if q]) - elif web_search_queries: - web_search_requests = len([q for q in web_search_queries if q]) - return web_search_requests + if not (grounding_metadata and isinstance(grounding_metadata, list)): + return None + unique_queries: Final = { + query + for grounding_metadata_item in grounding_metadata + for query in (grounding_metadata_item.get("webSearchQueries") or []) + if query + } + return len(unique_queries) or None @staticmethod def _create_streaming_choice( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d9a13ef1b98..a6da2c1fb09 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1428,7 +1428,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1465,7 +1465,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1502,7 +1502,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1539,7 +1539,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2933,7 +2933,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-10-19", @@ -2956,7 +2957,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", @@ -2987,7 +2989,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", @@ -3018,7 +3021,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { "supports_mid_conversation_system": true, @@ -3050,7 +3054,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5": { "supports_mid_conversation_system": true, @@ -3113,7 +3118,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-opus-4-1": { "deprecation_date": "2026-08-05", @@ -3135,7 +3141,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", @@ -3157,7 +3164,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-5": { "supports_mid_conversation_system": true, @@ -3188,7 +3196,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", @@ -3214,7 +3223,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -14724,7 +14734,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14746,7 +14757,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-1": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14768,7 +14780,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-5": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14791,7 +14804,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-6": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14814,7 +14828,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-7": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14916,7 +14931,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14960,7 +14976,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14983,7 +15000,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-5": { "cache_creation_input_token_cost": 3.74997e-06, @@ -33861,7 +33879,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -33881,7 +33900,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -33904,7 +33924,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, @@ -33930,7 +33951,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -33949,7 +33971,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -33970,7 +33993,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -33993,7 +34017,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -34011,7 +34036,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -34034,7 +34060,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "openrouter/anthropic/claude-opus-5": { "prompt_cache_min_tokens": 512, @@ -36501,7 +36528,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 4096 }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, @@ -36583,7 +36611,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/deepseek-ai/deepseek-v3": { "input_cost_per_token": 1.45e-06, @@ -36658,7 +36687,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, @@ -39687,7 +39717,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, @@ -39706,7 +39737,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.1": { "cache_creation_input_token_cost": 1.875e-05, @@ -39725,7 +39757,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -39745,7 +39778,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -39767,7 +39801,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -39786,7 +39821,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { "cache_creation_input_token_cost": 3.75e-06, @@ -39804,7 +39840,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, @@ -41169,7 +41206,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -41203,7 +41241,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -50107,7 +50146,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -50124,7 +50164,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { "max_tokens": 16384, @@ -50139,7 +50180,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { "max_tokens": 16384, @@ -50155,7 +50197,8 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_reasoning": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { "max_tokens": 16384, @@ -50170,7 +50213,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { "max_tokens": 16384, @@ -50481,6 +50525,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, @@ -50533,6 +50603,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 026a02d6b1d..0e3cbbfd560 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -23538,7 +23538,7 @@ "paths": { "/prompts": { "post": { - "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", + "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_data\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", "operationId": "create_prompt_prompts_post", "requestBody": { "content": { diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f2327b18914..c6d7975b75e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -14,6 +14,7 @@ import json import math import traceback from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protocol, TypeVar, cast @@ -494,8 +495,13 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: - """Create team member budget table with provided limits""" + """Create team member budget table with provided limits. + + The team's own reset period is only inherited when the caller left the + member duration out, so an explicit null means "never resets". + """ from litellm.proxy._types import BudgetNewRequest from litellm.proxy.management_endpoints.budget_management_endpoints import ( new_budget, @@ -509,7 +515,11 @@ class TeamMemberBudgetHandler: # Create budget request with all provided limits budget_request: Final = BudgetNewRequest( budget_id=budget_id, - budget_duration=data.budget_duration or team_member_budget_duration, + budget_duration=( + team_member_budget_duration + if "team_member_budget_duration" in explicitly_set_fields + else data.budget_duration or team_member_budget_duration + ), ) if team_member_budget is not None: @@ -545,8 +555,13 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: - """Upsert team member budget table with provided limits""" + """Upsert team member budget table with provided limits. + + A field the caller explicitly sent as null is written as null, so a + team can keep a member budget while dropping its reset period. + """ from litellm.proxy._types import BudgetNewRequest from litellm.proxy.management_endpoints.budget_management_endpoints import ( update_budget, @@ -560,14 +575,16 @@ class TeamMemberBudgetHandler: # Budget exists - create update request with only provided values budget_request: Final = BudgetNewRequest(budget_id=team_member_budget_id) - if team_member_budget is not None: + if team_member_budget is not None or "team_member_budget" in explicitly_set_fields: budget_request.max_budget = team_member_budget - if team_member_rpm_limit is not None: + if team_member_rpm_limit is not None or "team_member_rpm_limit" in explicitly_set_fields: budget_request.rpm_limit = team_member_rpm_limit - if team_member_tpm_limit is not None: + if team_member_tpm_limit is not None or "team_member_tpm_limit" in explicitly_set_fields: budget_request.tpm_limit = team_member_tpm_limit - if team_member_budget_duration is not None: + if team_member_budget_duration is not None or "team_member_budget_duration" in explicitly_set_fields: budget_request.budget_duration = team_member_budget_duration + if team_member_budget_duration is None: + budget_request.budget_reset_at = None budget_row: Final = await _as_budget_write(update_budget)( budget_obj=budget_request, @@ -593,6 +610,7 @@ class TeamMemberBudgetHandler: team_member_rpm_limit=team_member_rpm_limit, team_member_tpm_limit=team_member_tpm_limit, team_member_budget_duration=team_member_budget_duration, + explicitly_set_fields=explicitly_set_fields, ) # Remove team member fields from updated_kv @@ -1479,6 +1497,7 @@ async def new_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + explicitly_set_fields=data.model_fields_set, ) ## ADD TO TEAM TABLE @@ -2184,6 +2203,7 @@ async def update_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + explicitly_set_fields=_team_member_fields_in_request, ) # Backfill team_memberships for members who joined before the # budget was configured — they won't have a membership row yet. diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index a289ed7cbfb..e90adc163e2 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -323,6 +323,7 @@ def create_versioned_prompt_spec(db_prompt: _PromptRow) -> PromptSpec: prompt_info=prompt_info, created_at=row.created_at, updated_at=row.updated_at, + version=row.version, environment=row.environment, created_by=row.created_by, ) @@ -334,6 +335,21 @@ class Prompt(BaseModel): prompt_info: PromptInfo | None = None +AMBIGUOUS_PROMPT_DATA_ERROR: Final = ( + "litellm_params.prompt_id cannot be combined with prompt_data keyed by template name. " + 'Send a flat template, prompt_data={"content": "...", "metadata": {...}}, together with litellm_params.prompt_id, ' + 'or send prompt_data={"": {"content": "...", "metadata": {...}}} without litellm_params.prompt_id.' +) + + +def is_ambiguous_keyed_prompt_data(litellm_params: PromptLiteLLMParams) -> bool: + extra_fields: Final = litellm_params.model_extra or {} + prompt_data: Final = extra_fields.get("prompt_data") + if not litellm_params.prompt_id or not isinstance(prompt_data, dict): + return False + return bool(prompt_data) and "content" not in prompt_data + + class PatchPromptRequest(BaseModel): litellm_params: PromptLiteLLMParams | None = None prompt_info: PromptInfo | None = None @@ -737,11 +753,9 @@ async def create_prompt( -d '{ "prompt_id": "my_prompt", "litellm_params": { - "prompt_id": "json_prompt", + "prompt_id": "my_prompt", "prompt_integration": "dotprompt", - ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED - "prompt_directory": "/path/to/dotprompt/folder", - "prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}} + "prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}} }, "prompt_info": { "prompt_type": "config" @@ -763,6 +777,9 @@ async def create_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Extract environment from request environment: Final = ( @@ -857,6 +874,9 @@ async def update_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success") base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) @@ -1086,6 +1106,9 @@ async def patch_prompt( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if request.litellm_params is not None and is_ambiguous_keyed_prompt_data(request.litellm_params): + raise HTTPException(status_code=400, detail=AMBIGUOUS_PROMPT_DATA_ERROR) + try: # Resolve the target row: find the latest version in the given environment base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index 695bdabfe83..46caa30fdab 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -147,6 +147,9 @@ class InMemoryPromptRegistry: prompt_info=prompt.prompt_info or PromptInfo(prompt_type="config"), created_at=prompt.created_at, updated_at=prompt.updated_at, + version=prompt.version, + environment=prompt.environment, + created_by=prompt.created_by, ) # store references to the prompt in memory diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e55b254ab8e..9a3905da950 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6268,7 +6268,14 @@ class ProxyConfig: ): from litellm.utils import _update_dictionary - combined_router_settings = _update_dictionary(config_router_settings, db_router_settings.param_value) + db_overlay_deferring_empty_lists_to_config: Final = { + k: v + for k, v in db_router_settings.param_value.items() + if not (k in config_router_settings and isinstance(v, list) and len(v) == 0) + } + combined_router_settings = _update_dictionary( + config_router_settings, db_overlay_deferring_empty_lists_to_config + ) elif config_router_settings is not None and isinstance(config_router_settings, dict): combined_router_settings = config_router_settings elif db_router_settings is not None and isinstance(db_router_settings.param_value, dict): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d9a13ef1b98..a6da2c1fb09 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1428,7 +1428,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1465,7 +1465,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1502,7 +1502,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1539,7 +1539,7 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 512 }, "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2933,7 +2933,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-10-19", @@ -2956,7 +2957,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", @@ -2987,7 +2989,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", @@ -3018,7 +3021,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { "supports_mid_conversation_system": true, @@ -3050,7 +3054,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5": { "supports_mid_conversation_system": true, @@ -3113,7 +3118,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-opus-4-1": { "deprecation_date": "2026-08-05", @@ -3135,7 +3141,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", @@ -3157,7 +3164,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-5": { "supports_mid_conversation_system": true, @@ -3188,7 +3196,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", @@ -3214,7 +3223,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -14724,7 +14734,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14746,7 +14757,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-1": { "cache_creation_input_token_cost": 1.874999e-05, @@ -14768,7 +14780,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-opus-4-5": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14791,7 +14804,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-6": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14814,7 +14828,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 4096 }, "databricks/databricks-claude-opus-4-7": { "cache_creation_input_token_cost": 6.25002e-06, @@ -14916,7 +14931,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14960,7 +14976,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.74997e-06, @@ -14983,7 +15000,8 @@ "supports_legacy_thinking": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 1024 }, "databricks/databricks-claude-sonnet-5": { "cache_creation_input_token_cost": 3.74997e-06, @@ -33861,7 +33879,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -33881,7 +33900,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -33904,7 +33924,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, @@ -33930,7 +33951,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -33949,7 +33971,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -33970,7 +33993,8 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -33993,7 +34017,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -34011,7 +34036,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -34034,7 +34060,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "openrouter/anthropic/claude-opus-5": { "prompt_cache_min_tokens": 512, @@ -36501,7 +36528,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 4096 }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, @@ -36583,7 +36611,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/deepseek-ai/deepseek-v3": { "input_cost_per_token": 1.45e-06, @@ -36658,7 +36687,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_response_schema": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "prompt_cache_min_tokens": 1024 }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, @@ -39687,7 +39717,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, @@ -39706,7 +39737,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.1": { "cache_creation_input_token_cost": 1.875e-05, @@ -39725,7 +39757,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -39745,7 +39778,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -39767,7 +39801,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -39786,7 +39821,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { "cache_creation_input_token_cost": 3.75e-06, @@ -39804,7 +39840,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, @@ -41169,7 +41206,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -41203,7 +41241,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -50107,7 +50146,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -50124,7 +50164,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { "max_tokens": 16384, @@ -50139,7 +50180,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { "max_tokens": 16384, @@ -50155,7 +50197,8 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_reasoning": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { "max_tokens": 16384, @@ -50170,7 +50213,8 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { "max_tokens": 16384, @@ -50481,6 +50525,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, @@ -50533,6 +50603,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, diff --git a/ruff-tests.toml b/ruff-tests.toml index e036fb4946c..d75f10b9605 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -55,6 +55,40 @@ # never fires and a subprocess still reads the real keys the test believes it # cleared. The manual restore underneath is skipped whenever the body raises, # so every later test in that worker inherits a plain dict for an environment +# PGH005 an assertion on a mock attribute the library never defines. `assert +# m.called_once` and a bare `m.assert_called_once` both read as checks and +# neither is one: a Mock invents whatever attribute it is asked for, so the +# first is always truthy and the second is an attribute nobody calls +# F631 `assert (cond, "message")` asserts a two-element tuple, which is always +# truthy. The message meant to explain the failure is what stops the assertion +# from ever having one +# F634 `if (a, b):` branches on a tuple, so the branch is always taken and the +# condition it was written to test is never evaluated +# PT010 `pytest.raises()` with no exception type accepts anything the block raises, +# including the TypeError a refactor introduced +# PT030 the `pytest.warns` twin of PT011. `Warning` or `UserWarning` with no `match=` +# passes on any warning that broad +# PT031 the `pytest.warns` twin of PT012. Everything after the warning call is dead, +# so an `assert` sitting there is never checked +# B012 a `return`, `break` or `continue` inside `finally` discards whatever exception +# was in flight, so the AssertionError the test just raised is thrown away and +# the test reports green +# B013 a one-element tuple where the exception class was meant, which reads as a +# wider handler than it is +# B014 an exception named twice in one handler, or a subclass beside its parent. The +# second name does nothing, and it is usually the one someone meant to change +# B016 `raise "message"` raises a str, so the failure the test set up is replaced by +# a TypeError from the raise itself +# B022 `contextlib.suppress()` with no arguments suppresses nothing, so the call it +# wraps still raises +# B029 `except ():` catches nothing, so the recovery or skip written in that handler +# never happens +# B030 an `except` naming something that is not an exception class raises TypeError +# while unwinding, replacing the error under test +# F707 a bare `except:` ahead of another handler makes every handler below it +# unreachable +# PLE0704 a bare `raise` outside an except block raises RuntimeError instead of +# re-raising anything # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -83,4 +117,19 @@ lint.select = [ "B025", "F632", "B003", + "PGH005", + "F631", + "F634", + "PT010", + "PT030", + "PT031", + "B012", + "B013", + "B014", + "B016", + "B022", + "B029", + "B030", + "F707", + "PLE0704", ] diff --git a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py index b92ed13302e..3c10e0db8d7 100644 --- a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py +++ b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py @@ -577,3 +577,83 @@ async def test_dotprompt_with_prompt_version(): ) assert "Version 2:" in v2_rendered assert "Test v2" in v2_rendered + + +def test_keyed_prompt_data_with_prompt_id_keeps_real_content(): + prompt_data = { + "json_prompt": { + "content": "You are a pirate. Begin every reply with AHOY.", + "metadata": {"model": "gpt-4o-mini"}, + } + } + + manager = PromptManager(prompt_data=prompt_data, prompt_id="agent-prompt") + + template = manager.get_prompt("json_prompt") + assert template is not None + assert template.content == "You are a pirate. Begin every reply with AHOY." + assert template.model == "gpt-4o-mini" + assert "agent-prompt" not in manager.prompts + + +def test_flat_prompt_data_with_prompt_id_registers_under_prompt_id(): + manager = PromptManager( + prompt_data={"content": "Hello {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + prompt_id="flat-prompt", + ) + + template = manager.get_prompt("flat-prompt") + assert template is not None + assert template.content == "Hello {{name}}" + assert manager.render("flat-prompt", {"name": "world"}) == "Hello world" + + +def test_get_prompt_falls_back_to_base_id_for_versioned_id(): + manager = PromptManager( + prompt_data={"content": "Hi", "metadata": {}}, + prompt_id="my-prompt", + ) + + assert manager.get_prompt("my-prompt.v1") is not None + assert manager.get_prompt("my-prompt.v12") is not None + assert manager.get_prompt("my-prompt.vx") is None + assert manager.get_prompt("other-prompt.v1") is None + + +def test_should_run_prompt_management_accepts_versioned_id(): + from litellm.integrations.dotprompt import DotpromptManager + + dotprompt_manager = DotpromptManager( + prompt_data={"content": "Hi", "metadata": {}}, + prompt_id="versioned-prompt", + ) + + assert dotprompt_manager.should_run_prompt_management("versioned-prompt", None, {}) is True + assert dotprompt_manager.should_run_prompt_management("versioned-prompt.v1", None, {}) is True + assert dotprompt_manager.should_run_prompt_management("missing-prompt", None, {}) is False + + +def test_prompt_initializer_registers_flat_db_prompt_under_base_id(): + from litellm.integrations.dotprompt import DotpromptManager, prompt_initializer + from litellm.types.prompts.init_prompts import ( + PromptInfo, + PromptLiteLLMParams, + PromptSpec, + ) + + litellm_params = PromptLiteLLMParams( + prompt_integration="dotprompt", + prompt_data={"content": "AHOY {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + ) + prompt_spec = PromptSpec( + prompt_id="agent-prompt.v1", + litellm_params=litellm_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + + dotprompt_manager = prompt_initializer(litellm_params, prompt_spec) + + assert isinstance(dotprompt_manager, DotpromptManager) + template = dotprompt_manager.prompt_manager.get_prompt("agent-prompt") + assert template is not None + assert template.content == "AHOY {{name}}" diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index c2e2f92ad8a..ee2a31beff7 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -1,19 +1,41 @@ """Test health check helper functions""" +import struct +import zlib from unittest.mock import AsyncMock, MagicMock, patch import pytest - +import litellm from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME -from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers +from litellm.litellm_core_utils.health_check_helpers import ( + IMAGE_EDIT_HEALTH_CHECK_PROMPT, + HealthCheckHelpers, +) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS +def _png_chunks(png: bytes, offset: int = 8) -> tuple[tuple[bytes, bytes], ...]: + if offset >= len(png): + return () + (length,) = struct.unpack(">I", png[offset : offset + 4]) + chunk = (png[offset + 4 : offset + 8], png[offset + 8 : offset + 8 + length]) + return (chunk, *_png_chunks(png, offset + 12 + length)) + + +def _distinct_rgb_colors(png: bytes) -> set[bytes]: + width = int.from_bytes(png[16:20], "big") + raw = zlib.decompress(b"".join(data for tag, data in _png_chunks(png) if tag == b"IDAT")) + row_size = 1 + width * 3 + rows = tuple(raw[i : i + row_size] for i in range(0, len(raw), row_size)) + assert all(row[0] == 0 for row in rows) + return {bytes(row[i : i + 3]) for row in rows for i in range(1, row_size, 3)} + + @pytest.mark.asyncio -async def test_image_edit_health_check_handler_uses_png_and_prompt(): +async def test_image_edit_health_check_handler_uses_descriptive_prompt_and_multicolor_png(): model_params = {"model": "openai/gpt-image-1", "api_key": "sk-test"} mode_handlers = HealthCheckHelpers.get_mode_handlers( model="gpt-image-1", @@ -31,20 +53,76 @@ async def test_image_edit_health_check_handler_uses_png_and_prompt(): model="gpt-image-1", custom_llm_provider="openai", model_params=model_params, - prompt="edit this image", + prompt="test from litellm", )["image_edit"]() assert mock_aimage_edit.call_count == 2 - default_call = mock_aimage_edit.call_args_list[0].kwargs - explicit_call = mock_aimage_edit.call_args_list[1].kwargs - assert default_call["model"] == "openai/gpt-image-1" - assert default_call["prompt"] == "test" - assert explicit_call["prompt"] == "edit this image" - image = default_call["image"] + for handler_call in mock_aimage_edit.call_args_list: + assert handler_call.kwargs["model"] == "openai/gpt-image-1" + assert handler_call.kwargs["prompt"] == IMAGE_EDIT_HEALTH_CHECK_PROMPT + image = mock_aimage_edit.call_args_list[0].kwargs["image"] assert isinstance(image, bytes) assert image.startswith(b"\x89PNG") assert int.from_bytes(image[16:20], "big") == 512 assert int.from_bytes(image[20:24], "big") == 512 + assert len(_distinct_rgb_colors(image)) >= 2 + + +@pytest.mark.asyncio +async def test_ahealth_check_image_edit_treats_content_policy_violation_as_healthy(): + moderation_error = litellm.ContentPolicyViolationError( + message="Your request was rejected as a result of our safety system.", + model="gpt-image-1", + llm_provider="openai", + ) + with patch( # test-quality-ok: the public health-check path has no dependency injection seam + "litellm.aimage_edit", new_callable=AsyncMock, side_effect=moderation_error + ): + result = await ahealth_check( + {"model": "gpt-image-1", "api_key": "sk-test"}, + mode="image_edit", + ) + + assert "error" not in result + + +@pytest.mark.asyncio +async def test_ahealth_check_image_edit_treats_moderation_blocked_code_as_healthy(): + moderation_blocked = litellm.BadRequestError( + message=( + '{"error": {"code": "moderation_blocked", "message": "Your request was blocked", ' + '"moderation_stage": "output", "type": "invalid_request_error"}}' + ), + model="gpt-image-1", + llm_provider="openai", + ) + with patch( # test-quality-ok: the public health-check path has no dependency injection seam + "litellm.aimage_edit", new_callable=AsyncMock, side_effect=moderation_blocked + ): + result = await ahealth_check( + {"model": "gpt-image-1", "api_key": "sk-test"}, + mode="image_edit", + ) + + assert "error" not in result + + +@pytest.mark.asyncio +async def test_ahealth_check_image_edit_still_fails_on_non_moderation_errors(): + auth_error = litellm.AuthenticationError( + message="Incorrect API key provided", + llm_provider="openai", + model="gpt-image-1", + ) + with patch( # test-quality-ok: the public health-check path has no dependency injection seam + "litellm.aimage_edit", new_callable=AsyncMock, side_effect=auth_error + ): + result = await ahealth_check( + {"model": "gpt-image-1", "api_key": "sk-bad"}, + mode="image_edit", + ) + + assert "error" in result @pytest.mark.asyncio @@ -88,9 +166,7 @@ def test_update_model_params_with_health_check_tracking_information(): # Verify that litellm_metadata was added assert "litellm_metadata" in result - assert result["litellm_metadata"]["tags"] == [ - LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME - ] + assert result["litellm_metadata"]["tags"] == [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME] # Verify the auth setup was called mock_add_auth.assert_called_once() @@ -169,16 +245,12 @@ async def test_ahealth_check_failure_masks_raw_request_headers(): if "Authorization" in headers: auth_header = headers["Authorization"] # Should be masked (e.g., "Be****90" or similar) - assert ( - auth_header != f"Bearer {test_api_key}" - ), "Authorization header must be masked" - assert ( - auth_header != test_api_key - ), "API key must not appear in Authorization header" + assert auth_header != f"Bearer {test_api_key}", "Authorization header must be masked" + assert auth_header != test_api_key, "API key must not appear in Authorization header" # Masked headers typically have asterisks or are truncated - assert "*" in auth_header or len(auth_header) < len( - f"Bearer {test_api_key}" - ), f"Authorization header should be masked but got: {auth_header}" + assert "*" in auth_header or len(auth_header) < len(f"Bearer {test_api_key}"), ( + f"Authorization header should be masked but got: {auth_header}" + ) # Content-Type should remain unmasked (not sensitive) if "Content-Type" in headers: @@ -257,9 +329,7 @@ async def test_batch_health_check_skips_bridge_when_no_logging_obj(): "litellm_metadata": litellm_metadata, } - with patch( - "litellm.alist_batches", new_callable=AsyncMock, return_value={} - ) as mock_alist: + with patch("litellm.alist_batches", new_callable=AsyncMock, return_value={}) as mock_alist: await HealthCheckHelpers._batch_health_check( custom_llm_provider="openai", model_params={"model": "openai/gpt-4"}, @@ -283,9 +353,7 @@ async def test_batch_health_check_uses_alist_batches_for_supported_providers(): "litellm_metadata": litellm_metadata, } - with patch( - "litellm.alist_batches", new_callable=AsyncMock, return_value={} - ) as mock_alist: + with patch("litellm.alist_batches", new_callable=AsyncMock, return_value={}) as mock_alist: await HealthCheckHelpers._batch_health_check( custom_llm_provider=provider, model_params={"model": f"{provider}/some-model"}, @@ -344,9 +412,7 @@ async def test_realtime_health_check_uses_model_level_vertex_params(): fake_vertex_base = MagicMock() fake_vertex_base.get_vertex_region = MagicMock(return_value="us-central1") - fake_vertex_base._ensure_access_token_async = AsyncMock( - return_value=("model-level-token", "model-level-project") - ) + fake_vertex_base._ensure_access_token_async = AsyncMock(return_value=("model-level-token", "model-level-project")) connect_calls = [] with ( @@ -381,8 +447,7 @@ async def test_realtime_health_check_uses_model_level_vertex_params(): custom_llm_provider="vertex_ai", ) assert connect_calls[0]["url"] == ( - "wss://us-central1-aiplatform.googleapis.com/ws/" - "google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" + "wss://us-central1-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" ) assert connect_calls[0]["additional_headers"] == { "Authorization": "Bearer model-level-token", diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 177c71eeae7..18d1aa949a8 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2757,3 +2757,176 @@ def test_video_generation_with_input_reference_keeps_file_multipart(): "seconds": "4", } assert result.status == "queued" + + +AZURE_AI_BASE = "https://myfoundry.services.ai.azure.com" +AZURE_AI_CHAT_COMPLETIONS_URL = f"{AZURE_AI_BASE}/models/chat/completions" + +def _a_tool_with_an_unsupported_field() -> dict: + return { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, + "strict": True, + } + +A_COMPLETION = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "grok-3", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + +TOOL_LEVEL_REJECTION = "Extra inputs are not permitted: tools[0].strict" +UNRELATED_REJECTION = "Extra inputs are not permitted: temperature" +A_REJECTION_THE_PROVIDER_CANNOT_FIX = "The model is not available in this region" + + +class _RecordedAzureAI: + def __init__(self, responses: list[httpx.Response]) -> None: + self._responses = responses + self.bodies: list[dict] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.bodies.append(json.loads(request.content)) + return self._responses[min(len(self.bodies) - 1, len(self._responses) - 1)] + + +@pytest.fixture +def httpx_transport(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +def _rejection(message: str) -> httpx.Response: + return httpx.Response(422, json={"error": {"message": message}}) + + +def _call_azure_ai(recorder: _RecordedAzureAI, **overrides): + import respx + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + return litellm.completion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + **overrides, + ) + + +def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried(): + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + response = _call_azure_ai(recorder) + + assert len(recorder.bodies) == 2 + assert recorder.bodies[0]["tools"][0]["strict"] is True + assert "strict" not in recorder.bodies[1]["tools"][0] + assert response.choices[0].message.content == "sent" + + +def test_the_retry_changes_only_the_field_the_provider_named(): + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + _call_azure_ai(recorder) + + first, second = recorder.bodies + assert second["messages"] == first["messages"] + assert second["model"] == first["model"] + assert second["tools"][0]["function"] == first["tools"][0]["function"] + + +def test_a_provider_that_keeps_rejecting_is_not_retried_forever(): + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)]) + + with pytest.raises(litellm.BadRequestError) as raised: + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 2 + assert raised.value.status_code == 422 + + +def test_a_rejection_the_provider_cannot_fix_is_not_retried_at_all(): + recorder = _RecordedAzureAI([_rejection(A_REJECTION_THE_PROVIDER_CANNOT_FIX)]) + + with pytest.raises(litellm.BadRequestError): + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 1 + + +def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was_asked_for(): + recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION)]) + + with pytest.raises(litellm.BadRequestError): + _call_azure_ai(recorder) + + assert len(recorder.bodies) == 1 + + +def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for(): + recorder = _RecordedAzureAI( + [_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + response = _call_azure_ai(recorder, drop_params=True) + + assert len(recorder.bodies) == 2 + assert response.choices[0].message.content == "sent" + + +@pytest.mark.asyncio +async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_async_path( + httpx_transport, +): + import respx + + recorder = _RecordedAzureAI( + [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] + ) + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + response = await litellm.acompletion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + ) + + assert len(recorder.bodies) == 2 + assert recorder.bodies[0]["tools"][0]["strict"] is True + assert "strict" not in recorder.bodies[1]["tools"][0] + assert response.choices[0].message.content == "sent" + + +@pytest.mark.asyncio +async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_async_path( + httpx_transport, +): + import respx + + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)]) + + with respx.mock(assert_all_called=True) as router: + router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) + with pytest.raises(litellm.BadRequestError): + await litellm.acompletion( + model="azure_ai/grok-3", + messages=[{"role": "user", "content": "hi"}], + tools=[_a_tool_with_an_unsupported_field()], + api_base=AZURE_AI_BASE, + api_key="fake-key", + ) + + assert len(recorder.bodies) == 2 diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py index fa6f23dc7ff..3f93264d0d0 100644 --- a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -1,3 +1,4 @@ +import litellm from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig @@ -108,6 +109,284 @@ def test_thinking_mode_active_bool_thinking_returns_false_without_crashing(): assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False +class TestDeepSeekVisionMultimodalContent: + """Image content lists are forwarded only for user messages on vision models.""" + + VISION_MODEL = "deepseek/deepseek-v4-flash-vision-exp" + NON_VISION_MODEL = "deepseek/deepseek-chat" + + def setup_method(self): + self.config = DeepSeekChatConfig() + prior_entry = litellm.model_cost.get(self.VISION_MODEL) + self._prior_registry_entry = dict(prior_entry) if prior_entry is not None else None + litellm.register_model( + { + "deepseek/deepseek-v4-flash-vision-exp": { + "litellm_provider": "deepseek", + "mode": "chat", + "input_cost_per_token": 4.4e-07, + "output_cost_per_token": 1.32e-06, + "supports_vision": True, + } + } + ) + + def teardown_method(self): + if self._prior_registry_entry is None: + litellm.model_cost.pop(self.VISION_MODEL, None) + else: + litellm.model_cost[self.VISION_MODEL] = self._prior_registry_entry + + @staticmethod + def _image_message(role="user"): + return { + "role": role, + "content": [ + {"type": "text", "text": "what is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg", "detail": "auto"}, + }, + ], + } + + def test_user_image_list_forwarded_on_vision_model(self): + result = self.config._transform_messages([self._image_message()], model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], list) + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][1]["type"] == "image_url" + assert result[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg" + + def test_image_list_collapsed_on_non_vision_model(self): + result = self.config._transform_messages([self._image_message()], model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "what is in this image?" + + def test_image_list_collapsed_on_non_user_roles_even_on_vision_model(self): + for role in ("assistant", "system"): + result = self.config._transform_messages([self._image_message(role=role)], model=self.VISION_MODEL) + + assert result[0]["content"] == "what is in this image?" + + def test_audio_block_collapsed_even_on_vision_model(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "transcribe this"}, + {"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "transcribe this" + + def test_typeless_image_block_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this"}, + {"image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "what is this" + + def test_text_only_content_list_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello "}, + {"type": "text", "text": "world"}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], str) + assert result[0]["content"] == "Hello world" + + def test_search_results_text_appended_on_forwarded_message(self): + message = self._image_message() + message["search_results"] = [{"source": "kb", "content": [{"text": "article body"}]}] + + result = self.config._transform_messages([message], model=self.VISION_MODEL) + + content = result[0]["content"] + assert isinstance(content, list) + assert content[-1] == {"type": "text", "text": "kbarticle body"} + assert any(block.get("type") == "image_url" for block in content) + assert "search_results" not in result[0] + + def test_search_results_text_kept_on_collapse(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "context: "}], + "search_results": [{"source": "kb", "content": [{"text": "article body"}]}], + } + ] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "context: kbarticle body" + + def test_responses_shape_blocks_collapse_even_on_vision_model(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "what is this?" + + def test_image_block_missing_payload_collapses(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "hi"}, {"type": "image_url"}], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_image_block_empty_payload_object_collapses(self): + for payload in ({}, {"url": ""}, {"detail": "auto"}, None, 42): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "hi"}, {"type": "image_url", "image_url": payload}], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_image_block_string_payload_forwarded(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": "https://example.com/image.jpg"}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + content = result[0]["content"] + assert isinstance(content, list) + assert content[1]["image_url"] == {"url": "https://example.com/image.jpg"} + + def test_text_block_missing_text_field_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "text"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_string_content_search_results_folded_into_string(self): + messages = [ + { + "role": "tool", + "tool_call_id": "call_1", + "content": "summarize the docs", + "search_results": [{"source": "kb", "content": [{"text": "article body"}]}], + } + ] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "summarize the docskbarticle body" + + def test_plain_string_content_message_unchanged(self): + messages = [{"role": "user", "content": "hello"}] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0] is messages[0] + + def test_empty_content_list_untouched(self): + messages = [{"role": "user", "content": []}] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == [] + + def test_later_messages_still_collapsed_after_forwarded_one(self): + messages = [ + self._image_message(), + { + "role": "user", + "content": [ + {"type": "text", "text": "and "}, + {"type": "text", "text": "then?"}, + ], + }, + self._image_message(), + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], list) + assert result[1]["content"] == "and then?" + assert isinstance(result[2]["content"], list) + + def test_transform_request_preserves_image_url_block(self): + body = self.config.transform_request( + model=self.VISION_MODEL, + messages=[self._image_message()], + optional_params={}, + litellm_params={}, + headers={}, + ) + + content = body["messages"][0]["content"] + assert isinstance(content, list) + assert any(block.get("type") == "image_url" for block in content) + + async def test_async_transform_request_preserves_image_url_block(self): + body = await self.config.async_transform_request( + model=self.VISION_MODEL, + messages=[self._image_message()], + optional_params={}, + litellm_params={}, + headers={}, + ) + + content = body["messages"][0]["content"] + assert isinstance(content, list) + assert any(block.get("type") == "image_url" for block in content) + + class TestDeepSeekThinkingParams: """Test thinking and reasoning_effort parameter handling for DeepSeek.""" @@ -282,8 +561,6 @@ class TestDeepSeekThinkingParams: result = self.config._drop_unsupported_tools(optional_params) - assert result["tools"] == [ - {"type": "function", "function": {"name": "get_weather"}} - ] + assert result["tools"] == [{"type": "function", "function": {"name": "get_weather"}}] assert "tool_choice" not in result assert result["parallel_tool_calls"] is True diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 63a749dab84..95ec183792d 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -73,7 +73,13 @@ def test_validate_environment_sets_session_affinity_from_session_id(): assert headers["x-session-affinity"] == "session-id-123" -def test_validate_environment_sets_session_affinity_from_trace_id(): +def test_validate_environment_ignores_trace_id_for_session_affinity(): + """A trace id must not become the session id. + + litellm_trace_id defaults to a fresh uuid4 per request, so pinning + x-session-affinity to it sent every request to a different Fireworks node and + prompt caching never hit (cached_tokens stayed 0 across identical prompts). + """ config = FireworksAIConfig() headers = config.validate_environment( @@ -85,7 +91,25 @@ def test_validate_environment_sets_session_affinity_from_trace_id(): api_key="test-key", ) - assert headers["x-session-affinity"] == "trace-id-123" + assert "x-session-affinity" not in headers + + +def test_validate_environment_prefers_session_id_over_trace_id(): + config = FireworksAIConfig() + + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/test-model", + messages=[], + optional_params={}, + litellm_params={ + "litellm_session_id": "session-123", + "litellm_trace_id": "trace-id-123", + }, + api_key="test-key", + ) + + assert headers["x-session-affinity"] == "session-123" def test_validate_environment_does_not_set_session_affinity_without_session_id(): diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index deb148a07c0..42e330925a0 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1298,8 +1298,7 @@ def test_gemini_realtime_pipecat_ga_session_voice_and_tools(patch_gemini_audio_c assert len(messages) == 1 setup = json.loads(messages[0])["setup"] assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] - # Native-audio Live rejects speechConfig on setup (see _finalize_gemini_live_setup). - assert "speechConfig" not in setup.get("generationConfig", {}) + assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" assert setup["tools"][0]["function_declarations"][0]["name"] == "terminate_call" assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is False @@ -1843,20 +1842,6 @@ def test_is_audio_only_live_model_uses_cost_map(model, expected, patch_gemini_au assert GeminiRealtimeConfig._is_audio_only_live_model(model) == expected -@pytest.mark.parametrize( - "model,expected", - [ - ("gemini-2.5-flash-native-audio-latest", True), - ("gemini/gemini-2.5-flash-native-audio-latest", True), - ("gemini-3.1-flash-live-preview", False), - ("gemini/gemini-3.1-flash-live-preview", False), - ("gemini-2.0-flash", False), - ], -) -def test_is_native_audio_model_uses_cost_map(model, expected, patch_gemini_audio_cost_map_entries): - assert GeminiRealtimeConfig._is_native_audio_model(model) == expected - - def test_is_setup_message_and_is_content_message(): config = GeminiRealtimeConfig() assert config.is_setup_message({"setup": {}}) is True @@ -1865,3 +1850,17 @@ def test_is_setup_message_and_is_content_message(): assert config.is_content_message({"clientContent": {}}) is True assert config.is_content_message({"toolResponse": {}}) is True assert config.is_content_message({"setup": {}}) is False + + +def test_map_openai_params_drops_stock_voice_case_insensitively(): + """Regression: OpenAI stock voices are dropped regardless of casing so Gemini Live keeps its default voice. + + Non-OpenAI names pass through verbatim. + """ + cfg = GeminiRealtimeConfig() + + dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}) + assert "speechConfig" not in dropped.get("generationConfig", {}) + + passthrough = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Kore"}) + assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 790a32506f2..bfe12498dd6 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2,7 +2,7 @@ import asyncio import json import re from copy import deepcopy -from typing import List, cast +from typing import Final, List, cast from unittest.mock import MagicMock, patch import pytest @@ -5579,3 +5579,25 @@ def test_accumulated_json_async_end_of_stream_drains_buffered_value(): result = asyncio.run(iterator.__anext__()) assert result is not None assert result.choices[0].delta.content == "a" + + +def test_calculate_web_search_requests_counts_unique_queries(): + """Gemini 3 per_query billing charges per unique query executed, not per emitted string. + + Regression for #36377: duplicate webSearchQueries within and across grounding + metadata items must collapse to the distinct-query count, and empty strings must + be ignored, matching Google's documented Grounding-with-Search billing rule. + """ + duplicates_in_one_item: Final = [ + {"webSearchQueries": ["euro 2024 winner", "euro 2024 winner", "spain england final", ""]} + ] + assert VertexGeminiConfig._calculate_web_search_requests(duplicates_in_one_item) == 2 + + duplicates_across_items: Final = [ + {"webSearchQueries": ["euro 2024 winner"]}, + {"webSearchQueries": ["euro 2024 winner", "spain england final"]}, + ] + assert VertexGeminiConfig._calculate_web_search_requests(duplicates_across_items) == 2 + + assert VertexGeminiConfig._calculate_web_search_requests([]) is None + assert VertexGeminiConfig._calculate_web_search_requests([{"webSearchQueries": ["", ""]}]) is None diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 720c629cbf7..d4cf58bc0b4 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -15,7 +15,6 @@ from unittest.mock import AsyncMock, MagicMock import pytest import websockets.exceptions # registers websockets.exceptions on the websockets namespace - import litellm from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig @@ -278,7 +277,7 @@ async def test_vertex_realtime_text_in_text_out(): SERVER_TURN_COMPLETE, ] - async def _backend_recv(decode=True): # noqa: ARG001 + async def _backend_recv(decode=True): if not upstream_messages: # Signal normal connection close so the loop exits cleanly raise websockets.exceptions.ConnectionClosedOK(None, None) # type: ignore[arg-type] @@ -462,3 +461,98 @@ def test_vertex_function_call_output_omits_id(): assert "id" not in function_response assert function_response["name"] == "terminate_call" assert function_response["response"] == {"status": "ok"} + + +def test_vertex_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry): + """Regression: Vertex Live accepts speechConfig on native audio, so the client's voice must survive. + + Stripping it silently dropped voice selection for every Vertex native-audio + session. TEXT is still coerced away, which Vertex does reject. + """ + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + session_update = { + "type": "session.update", + "session": { + "output_modalities": ["text"], + "audio": {"output": {"voice": "Aoede"}}, + }, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede" + assert generation_config["responseModalities"] == ["AUDIO"] + + +def test_google_ai_studio_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry): + """Regression: AI Studio native-audio Live accepts speechConfig too, so the voice survives on both providers.""" + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + messages = GeminiRealtimeConfig().transform_realtime_request( + json.dumps( + { + "type": "session.update", + "session": { + "output_modalities": ["audio"], + "audio": {"output": {"voice": "Aoede"}}, + }, + } + ), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede" + + +def test_vertex_native_audio_drops_openai_stock_voice(patch_native_audio_cost_map_entry): + """Regression: OpenAI stock voice names must be dropped, not forwarded verbatim. + + Vertex Live closes the socket with 1007 on an unknown voice name, so a + client sending OpenAI's default voice would lose the session entirely. + Dropping the voice keeps the session alive on the model's default voice. + """ + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + session_update = { + "type": "session.update", + "session": {"audio": {"output": {"voice": "alloy"}}}, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert "speechConfig" not in generation_config + + +def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_cost_map_entry): + """A voice name outside the OpenAI stock set is forwarded verbatim so Gemini-native names keep working.""" + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + session_update = { + "type": "session.update", + "session": {"audio": {"output": {"voice": "Kore"}}}, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 6461245bb2c..ffa6bc601e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -2366,6 +2366,7 @@ async def test_update_team_team_member_budget_not_passed_to_db( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + explicitly_set_fields=frozenset(), ): # Remove team_member_budget from updated_kv as the real function does result_kv = updated_kv.copy() @@ -2738,6 +2739,138 @@ async def test_upsert_team_member_budget_table_no_existing_budget(): assert "team_member_budget_duration" not in result +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_clears_duration_kept_budget(mock_db_client): + """ + A request that keeps team_member_budget but explicitly nulls + team_member_budget_duration must clear the reset period and its reset time. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {"team_member_budget_id": "existing_budget_123"} + + mock_db_client.db.litellm_budgettable.update = AsyncMock( + side_effect=lambda where, data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv={ + "team_id": "test_team_id", + "team_member_budget": 100.0, + "team_member_budget_duration": None, + }, + team_member_budget=100.0, + team_member_budget_duration=None, + explicitly_set_fields={ + "team_member_budget", + "team_member_budget_duration", + }, + ) + + written = mock_db_client.db.litellm_budgettable.update.call_args.kwargs["data"] + assert written["max_budget"] == 100.0 + assert written["budget_duration"] is None + assert written["budget_reset_at"] is None + assert "rpm_limit" not in written + assert "tpm_limit" not in written + assert result["metadata"]["team_member_budget_id"] == "existing_budget_123" + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_explicit_null_duration_does_not_inherit_team_duration( + mock_db_client, +): + """ + A first-time member budget with an explicitly null duration must never + reset, even when the team itself has a reset period. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = "30d" + + mock_db_client.db.litellm_budgettable.create = AsyncMock( + side_effect=lambda data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=team_table, + new_team_data_json={"team_id": "test_team_id"}, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + team_member_budget_duration=None, + explicitly_set_fields={ + "team_member_budget", + "team_member_budget_duration", + }, + ) + + written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"] + assert written["max_budget"] == 100.0 + assert "budget_duration" not in written + assert "budget_reset_at" not in written + assert result["metadata"]["team_member_budget_id"] == written["budget_id"] + assert "team_member_budget" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_inherits_team_duration_when_duration_omitted( + mock_db_client, +): + """ + Omitting team_member_budget_duration keeps the existing inheritance of the + team's own reset period. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = "30d" + + mock_db_client.db.litellm_budgettable.create = AsyncMock( + side_effect=lambda data: SimpleNamespace(**data) + ) + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=team_table, + new_team_data_json={"team_id": "test_team_id"}, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + explicitly_set_fields={"team_member_budget"}, + ) + + written = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"] + assert written["budget_duration"] == "30d" + assert written["budget_reset_at"] is not None + assert result["metadata"]["team_member_budget_id"] == written["budget_id"] + + @pytest.mark.asyncio async def test_update_team_with_team_member_budget_duration( disable_audit_logging_for_mocked_team, @@ -2799,6 +2932,7 @@ async def test_update_team_with_team_member_budget_duration( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + explicitly_set_fields=frozenset(), ): result_kv = updated_kv.copy() result_kv.pop("team_member_budget", None) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py index 3e8e1e9dff8..67113ed9145 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py @@ -1,3 +1,4 @@ +import json import pytest from unittest.mock import MagicMock, AsyncMock, patch from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles @@ -246,3 +247,241 @@ async def test_patch_prompt_row_deleted_mid_update_returns_404(): exc_info.value.detail == "Prompt with ID test_prompt not found in environment development" ) + + +def test_is_ambiguous_keyed_prompt_data_shapes(): + from litellm.proxy.prompts.prompt_endpoints import is_ambiguous_keyed_prompt_data + + keyed_with_id = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + flat_with_id = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"content": "AHOY", "metadata": {}}, + ) + keyed_without_id = PromptLiteLLMParams( + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + no_prompt_data = PromptLiteLLMParams( + prompt_id="agent-prompt", prompt_integration="dotprompt" + ) + empty_prompt_data = PromptLiteLLMParams( + prompt_id="agent-prompt", prompt_integration="dotprompt", prompt_data={} + ) + + assert is_ambiguous_keyed_prompt_data(keyed_with_id) is True + assert is_ambiguous_keyed_prompt_data(flat_with_id) is False + assert is_ambiguous_keyed_prompt_data(keyed_without_id) is False + assert is_ambiguous_keyed_prompt_data(no_prompt_data) is False + assert is_ambiguous_keyed_prompt_data(empty_prompt_data) is False + + +@pytest.mark.asyncio +async def test_create_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + Prompt, + create_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = Prompt( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await create_prompt(request=request, user_api_key_dict=mock_user_auth) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +@pytest.mark.asyncio +async def test_patch_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + PatchPromptRequest, + patch_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = PatchPromptRequest( + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await patch_prompt( + prompt_id="agent-prompt", + request=request, + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +@pytest.mark.asyncio +async def test_patch_prompt_info_only_keeps_legacy_keyed_row_patchable(): + from litellm.proxy.prompts.prompt_endpoints import PatchPromptRequest, patch_prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + legacy_params = PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ) + target_row = MagicMock() + target_row.id = "row-1" + target_row.version = 1 + updated_row = MagicMock() + updated_row.model_dump.return_value = { + "prompt_id": "agent-prompt", + "version": 1, + "environment": "production", + "created_by": None, + "litellm_params": legacy_params.model_dump_json(), + "prompt_info": PromptInfo(prompt_type="db", environment="production").model_dump_json(), + } + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[target_row] + ) + mock_prisma_client.db.litellm_prompttable.update = AsyncMock(return_value=updated_row) + + existing_prompt = PromptSpec( + prompt_id="agent-prompt.v1", + litellm_params=legacy_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( # test-quality-ok: keeps the registry reload from touching global callback state + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry, + ): + mock_registry.get_prompt_by_id.return_value = existing_prompt + + await patch_prompt( + prompt_id="agent-prompt", + request=PatchPromptRequest(prompt_info=PromptInfo(prompt_type="db", environment="production")), + user_api_key_dict=mock_user_auth, + ) + + update_kwargs = mock_prisma_client.db.litellm_prompttable.update.await_args.kwargs + assert update_kwargs["where"] == {"id": "row-1"} + assert json.loads(update_kwargs["data"]["prompt_info"])["environment"] == "production" + assert json.loads(update_kwargs["data"]["litellm_params"])["prompt_data"] == { + "json_prompt": {"content": "AHOY", "metadata": {}} + } + + +@pytest.mark.asyncio +async def test_update_prompt_rejects_keyed_prompt_data_with_prompt_id(): + from fastapi import HTTPException + + from litellm.proxy.prompts.prompt_endpoints import ( + AMBIGUOUS_PROMPT_DATA_ERROR, + Prompt, + update_prompt, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN + ) + request = Prompt( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"json_prompt": {"content": "AHOY", "metadata": {}}}, + ), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): # test-quality-ok: proxy_server module global is the endpoint's only injection point + with pytest.raises(HTTPException) as exc_info: + await update_prompt( + prompt_id="agent-prompt", + request=request, + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == AMBIGUOUS_PROMPT_DATA_ERROR + + +def test_create_versioned_prompt_spec_populates_version(): + from litellm.proxy.prompts.prompt_endpoints import create_versioned_prompt_spec + + db_prompt = MagicMock() + db_prompt.model_dump.return_value = { + "prompt_id": "agent-prompt", + "version": 3, + "environment": "development", + "created_by": "user-1", + "litellm_params": { + "prompt_id": "agent-prompt", + "prompt_integration": "dotprompt", + }, + "prompt_info": {"prompt_type": "db"}, + "created_at": None, + "updated_at": None, + } + + prompt_spec = create_versioned_prompt_spec(db_prompt=db_prompt) + + assert prompt_spec.prompt_id == "agent-prompt.v3" + assert prompt_spec.version == 3 + + +def test_initialize_prompt_keeps_version_and_created_by(): + import litellm + from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry + + registry = InMemoryPromptRegistry() + prompt_spec = PromptSpec( + prompt_id="agent-prompt.v3", + litellm_params=PromptLiteLLMParams( + prompt_id="agent-prompt", + prompt_integration="dotprompt", + prompt_data={"content": "AHOY", "metadata": {}}, + ), + prompt_info=PromptInfo(prompt_type="db"), + version=3, + environment="development", + created_by="user-1", + ) + + with patch.object(litellm.logging_callback_manager, "add_litellm_callback"): # test-quality-ok: keeps initialize_prompt from registering a global callback that would leak across tests + initialized_prompt = registry.initialize_prompt(prompt=prompt_spec) + + assert initialized_prompt is not None + assert initialized_prompt.version == 3 + assert initialized_prompt.created_by == "user-1" + assert initialized_prompt.environment == "development" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index afc42e8db45..fc9d0a96163 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -4741,6 +4741,90 @@ async def test_add_router_settings_from_db_config_merge_logic(): assert combined_settings["nested_config"] == expected_nested +@pytest.mark.asyncio +async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks(): + """ + Regression test for DB router_settings rows carrying explicit empty lists + (e.g. {"fallbacks": []} written by the dashboard's delete-last-fallback flow): + empty lists are "no value" and must not clobber config.yaml fallbacks, + matching _deep_merge_dicts semantics. Non-empty DB lists still win. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_router = MagicMock() + mock_router.update_settings = MagicMock() + + config_data = { + "router_settings": { + "fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + "context_window_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + "content_policy_fallbacks": [{"gpt-oss-120b": ["granite-4-h-small"]}], + } + } + + mock_db_config = MagicMock() + mock_db_config.param_value = { + "fallbacks": [], + "context_window_fallbacks": [], + "content_policy_fallbacks": [{"gpt-oss-120b": ["other-model"]}], + "model_group_alias": {}, + "num_retries": 3, + } + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config) + + await proxy_config._add_router_settings_from_db_config( + config_data=config_data, + llm_router=mock_router, + prisma_client=mock_prisma_client, + ) + + combined_settings = mock_router.update_settings.call_args.kwargs + assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["other-model"]}] + assert combined_settings["num_retries"] == 3 + + +@pytest.mark.asyncio +async def test_add_router_settings_from_db_config_empty_db_list_still_clears_unconfigured_key(): + """ + An empty DB list only yields to config.yaml where the yaml configures that key. + When the yaml router_settings has no fallbacks, a DB {"fallbacks": []} (the + dashboard's delete-last-fallback write) must still reach the router so the + running pods drop the deleted fallback without a restart. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_router = MagicMock() + mock_router.update_settings = MagicMock() + + config_data = {"router_settings": {"num_retries": 1}} + + mock_db_config = MagicMock() + mock_db_config.param_value = {"fallbacks": [], "model_group_alias": {}} + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config) + + await proxy_config._add_router_settings_from_db_config( + config_data=config_data, + llm_router=mock_router, + prisma_client=mock_prisma_client, + ) + + combined_settings = mock_router.update_settings.call_args.kwargs + assert combined_settings["fallbacks"] == [] + assert combined_settings["num_retries"] == 1 + + @pytest.mark.asyncio async def test_add_router_settings_from_db_config_edge_cases(): """ @@ -11304,9 +11388,7 @@ class TestRouterModelNameOnStreamingChunks: with patch.object(ProxyLogging, "_fire_deferred_stream_logging"): return [ data - async for data in async_data_generator( - mock_response, MagicMock(spec=UserAPIKeyAuth), request_data - ) + async for data in async_data_generator(mock_response, MagicMock(spec=UserAPIKeyAuth), request_data) ] @staticmethod diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 82ab3b3bd08..dc2fbe3ed73 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3909,6 +3909,167 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_ assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9) +def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map): + """A router-facing model_name alias containing "/" whose leading segment is NOT a + registered provider must not be double-prefixed into a non-existent cost key. + + Regression test for #38069: alias "vertex/claude-opus-5" (real deployment + "vertex_ai/claude-opus-5") was re-prefixed into "vertex_ai/vertex/claude-opus-5", + silently pricing every streamed request at $0. + """ + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert selected == "vertex_ai/claude-opus-5" + + +def test_select_model_name_strips_duplicated_region_segment(_local_model_cost_map): + """A "region/model" alias whose leading segment repeats the request's region must + resolve to the region-priced cost key instead of keeping the region segment twice.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="us-east-1/anthropic.claude-v2:1", + ) + response._hidden_params = {"region_name": "us-east-1"} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="bedrock", + ) + + assert selected == "bedrock/us-east-1/anthropic.claude-v2:1" + + +def test_completion_cost_nonzero_for_slash_alias_model_name(_local_model_cost_map): + """End-to-end cost through a "/"-containing alias must price above zero (#38069).""" + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {"custom_llm_provider": "vertex_ai"} + response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50) + + cost = litellm.completion_cost( + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert cost == pytest.approx(100 * 5e-6 + 50 * 2.5e-5, rel=1e-9) + + +def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map): + """An alias that resolves to no known cost key keeps the legacy double-prefixed name.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="team/nonsense-model", + ) + response._hidden_params = {} + + selected = _select_model_name_for_cost_calc( + model=None, + completion_response=response, + custom_llm_provider="vertex_ai", + ) + + assert selected == "vertex_ai/team/nonsense-model" + + +def test_completion_cost_keeps_custom_priced_slash_router_id(_local_model_cost_map): + """A custom-priced router id containing "/" keeps its custom pricing instead of being + rewritten to the built-in key its suffix happens to match.""" + + from litellm.cost_calculator import _select_model_name_for_cost_calc + + litellm.register_model( + model_cost={ + "vertex/claude-opus-5": { + "input_cost_per_token": 7e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "vertex_ai", + } + } + ) + + selected = _select_model_name_for_cost_calc( + model="vertex_ai/claude-opus-5", + completion_response=None, + custom_pricing=True, + custom_llm_provider="vertex_ai", + router_model_id="vertex/claude-opus-5", + ) + assert selected == "vertex_ai/vertex/claude-opus-5" + + response = litellm.ModelResponse( + id="x", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + model="vertex/claude-opus-5", + ) + response._hidden_params = {"custom_llm_provider": "vertex_ai"} + response.usage = litellm.Usage(prompt_tokens=100, completion_tokens=50) + + cost = litellm.completion_cost( + completion_response=response, + custom_llm_provider="vertex_ai", + custom_pricing=True, + router_model_id="vertex/claude-opus-5", + ) + assert cost == pytest.approx(100 * 7e-6 + 50 * 8e-6, rel=1e-9) + + @pytest.mark.parametrize( ("model", "expected_1hr_rate"), [("claude-3-haiku-20240307", 5e-07), ("claude-3-opus-20240229", 3e-05)], diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b283d777524..366e510a94d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4452,14 +4452,101 @@ def test_get_prompt_cache_min_tokens_resolves_per_model( assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens -def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None: - """The same model can carry a different minimum per platform, so the threshold must come from - the platform's own cost-map entry rather than being derived from the model family name.""" - assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512 - assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024 - assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens( - model="anthropic.claude-fable-5" - ) +def test_get_prompt_cache_min_tokens_uniform_for_fable_5_across_platforms(local_model_cost_map: None) -> None: + """Anthropic removed the Amazon Bedrock override for Claude Fable 5, so its 512-token minimum + now applies on every platform. The Bedrock entries carried the old 1024 and the re-export + entries carried nothing, so the router judged 512-1023-token prefixes uncacheable and skipped + prompt-cache-affinity routing for prompts the provider demonstrably caches (issue #35011).""" + wrong: Final = { + model: get_prompt_cache_min_tokens(model=model) + for model, info in litellm.model_cost.items() + if "fable-5" in model + and info.get("supports_prompt_caching") + and get_prompt_cache_min_tokens(model=model) != 512 + } + assert not wrong, f"every Claude Fable 5 entry must carry prompt_cache_min_tokens 512: {wrong}" + + +ANTHROPIC_REEXPORT_CACHE_MIN: Final = { + "azure_ai/claude-fable-5": 512, + "azure_ai/claude-haiku-4-5": 4096, + "azure_ai/claude-opus-4-1": 1024, + "azure_ai/claude-opus-4-5": 4096, + "azure_ai/claude-opus-4-6": 4096, + "azure_ai/claude-opus-4-7": 2048, + "azure_ai/claude-opus-4-8": 1024, + "azure_ai/claude-sonnet-4-5": 1024, + "azure_ai/claude-sonnet-4-6": 1024, + "azure_ai/claude-sonnet-5": 1024, + "databricks/databricks-claude-haiku-4-5": 4096, + "databricks/databricks-claude-opus-4": 1024, + "databricks/databricks-claude-opus-4-1": 1024, + "databricks/databricks-claude-opus-4-5": 4096, + "databricks/databricks-claude-opus-4-6": 4096, + "databricks/databricks-claude-sonnet-4": 1024, + "databricks/databricks-claude-sonnet-4-5": 1024, + "databricks/databricks-claude-sonnet-4-6": 1024, + "openrouter/anthropic/claude-haiku-4.5": 4096, + "openrouter/anthropic/claude-opus-4": 1024, + "openrouter/anthropic/claude-opus-4.1": 1024, + "openrouter/anthropic/claude-opus-4.5": 4096, + "openrouter/anthropic/claude-opus-4.6": 4096, + "openrouter/anthropic/claude-opus-4.7": 2048, + "openrouter/anthropic/claude-sonnet-4": 1024, + "openrouter/anthropic/claude-sonnet-4.5": 1024, + "openrouter/anthropic/claude-sonnet-4.6": 1024, + "replicate/anthropic/claude-4-sonnet": 1024, + "replicate/anthropic/claude-4.5-haiku": 4096, + "replicate/anthropic/claude-4.5-sonnet": 1024, + "snowflake/claude-4-opus": 1024, + "snowflake/claude-4-sonnet": 1024, + "snowflake/claude-haiku-4-5": 4096, + "snowflake/claude-sonnet-4-5": 1024, + "snowflake/claude-sonnet-4-6": 1024, + "vercel_ai_gateway/anthropic/claude-haiku-4.5": 4096, + "vercel_ai_gateway/anthropic/claude-opus-4": 1024, + "vercel_ai_gateway/anthropic/claude-opus-4.1": 1024, + "vercel_ai_gateway/anthropic/claude-opus-4.5": 4096, + "vercel_ai_gateway/anthropic/claude-opus-4.6": 4096, + "vercel_ai_gateway/anthropic/claude-sonnet-4": 1024, + "vercel_ai_gateway/anthropic/claude-sonnet-4.5": 1024, + "vertex_ai/claude-fable-5": 512, + "vertex_ai/claude-fable-5@default": 512, +} + + +def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local_model_cost_map: None) -> None: + """Regression for issue #35011: these re-export entries carried no prompt_cache_min_tokens, so + they silently inherited the 1024 default. That skipped cache-affinity routing for Fable 5's + 512-1023-token prefixes and reported 1024-4095-token prompts as cacheable on the 2048/4096 + models. The entry must be explicit so a default change can never re-break them, which is why + this asserts the cost-map value itself and not just the resolver's answer.""" + wrong: Final = { + model: (litellm.model_cost[model].get("prompt_cache_min_tokens"), get_prompt_cache_min_tokens(model=model)) + for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items() + if litellm.model_cost[model].get("prompt_cache_min_tokens") != expected + or get_prompt_cache_min_tokens(model=model) != expected + } + assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}" + + +def test_anthropic_reexport_cache_minimums_present_in_root_cost_map() -> None: + """The root map ships to the CDN independently of the bundled backup, so both must carry the + minimum or proxies reading one of them regress to the 1024 default.""" + root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") + with open(root_map_path) as f: + root_map: Final = json.load(f) + wrong: Final = { + model: root_map[model].get("prompt_cache_min_tokens") + for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items() + if root_map[model].get("prompt_cache_min_tokens") != expected + } + fable_5_wrong: Final = { + model: info.get("prompt_cache_min_tokens") + for model, info in root_map.items() + if "fable-5" in model and info.get("supports_prompt_caching") and info.get("prompt_cache_min_tokens") != 512 + } + assert not wrong and not fable_5_wrong, f"root cost map diverges: {wrong | fable_5_wrong}" GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( diff --git a/type-discipline-budget.json b/type-discipline-budget.json index e0c0dd8147b..b61a0acd214 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16621 + "limit": 16620 }, "LIT011": { "limit": 5585 diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 78e5b572492..71bce0b254c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -17,15 +17,14 @@ import { ReasoningEffort, TierModelParamsByTier, pruneTierModelParams, - resolveComplexityDefaultModel, setTierModelReasoningEffort, - tierOptions, } from "./complexity_router_tiers"; import TierModelEffortRows from "./TierModelEffortRows"; import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import { type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; @@ -37,12 +36,12 @@ export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; -export interface ComplexityTiers { +export type ComplexityTiers = { SIMPLE: string[]; MEDIUM: string[]; COMPLEX: string[]; REASONING: string[]; -} +}; export type ClassificationRubric = "legacy" | "agentic" | "chat" | "business"; @@ -224,10 +223,6 @@ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label; -/** Tiers the plan-mode floor may name: the backend rejects a floor whose tier has no models. */ -export const planModeEligibleTiers = (tiers: ComplexityTiers): Array => - TIER_KEYS.filter((tier) => (tiers[tier] ?? []).length > 0); - const ComplexityRouterConfig: React.FC = ({ modelInfo, value, @@ -246,12 +241,12 @@ const ComplexityRouterConfig: React.FC = ({ onEscalationKeywordsChange, showValidationErrors = false, }) => { - const planModeTiers = planModeEligibleTiers(value.tiers); - const planModeTierOptions = tierOptions(value.tier_labels).filter((option) => - (planModeTiers as string[]).includes(option.value), - ); - const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers); - const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model); + const tierRows = activeTierRows(value); + const planModeTierOptions = tierRows + .filter((row) => row.models.length > 0) + .map((row) => ({ value: row.id, label: effectiveTierLabel(row.id as keyof ComplexityTiers, value.tier_labels) })); + const derivedDefaultModel = resolveComplexityDefaultModel(value); + const defaultModel = resolveComplexityDefaultModel(value, value.default_model); // An absent list means the proxy does not send the field yet, so every level is offered as before. // An empty list is the group's own answer that its deployments share no level, and is left empty. @@ -325,12 +320,13 @@ const ComplexityRouterConfig: React.FC = ({ - {TIER_KEYS.map((tier, index) => { + {tierRows.map((row: TierRow, index) => { + const tier = row.id as keyof ComplexityTiers; const tierInfo = TIER_DESCRIPTIONS[tier]; const label = effectiveTierLabel(tier, value.tier_labels); - const tierMissing = showValidationErrors && value.tiers[tier].length === 0; + const tierMissing = showValidationErrors && row.models.length === 0; return ( -
+
{index > 0 && }
@@ -339,7 +335,7 @@ const ComplexityRouterConfig: React.FC = ({ - Tier {index + 1} of {TIER_KEYS.length} · {tier} + Tier {index + 1} of {tierRows.length} · {row.id}
Examples: {tierInfo.examples} @@ -364,7 +360,7 @@ const ComplexityRouterConfig: React.FC = ({ handleTierChange(tier, models)} placeholder={`Select model(s) for ${label.toLowerCase()} queries`} emptyText="No models found" @@ -372,12 +368,12 @@ const ComplexityRouterConfig: React.FC = ({ /> handleTierModelEffortChange(tier, model, effort)} /> - {value.tiers[tier].length > 1 && ( + {row.models.length > 1 && ( Multiple models selected — the router randomly picks among them per request (or Thompson-samples within the pool when adaptive routing is on). @@ -483,9 +479,12 @@ const ComplexityRouterConfig: React.FC = ({
- onChange({ ...value, plan_mode_min_tier: enabled ? planModeTiers.at(-1) : undefined }) + onChange({ + ...value, + plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined, + }) } aria-label="Route plan-mode requests to a minimum tier" /> @@ -494,7 +493,7 @@ const ComplexityRouterConfig: React.FC = ({ Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. The classifier still wins when it picks higher, and the override only lasts while plan mode is active. - {planModeTiers.length === 0 && " Add models to a tier to enable this."} + {planModeTierOptions.length === 0 && " Add models to a tier to enable this."} {value.plan_mode_min_tier !== undefined && (
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 98ee2b7ae7c..87c4754f56d 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -22,7 +22,6 @@ import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, - ComplexityTiers, DEFAULT_ADAPTIVE_WEIGHTS, DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, @@ -40,7 +39,9 @@ import { getSemanticConfigError, getTierLabelsError, } from "./build_complexity_router_config"; -import { resolveComplexityDefaultModel } from "./complexity_router_tiers"; +import { activeTierName, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; +import { DEFAULT_TIER_LABELS } from "./complexity_router_tiers"; +import type { ComplexityTier } from "./KeywordTierRules"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import AutoRouterRoutingTest from "./AutoRouterRoutingTest"; @@ -104,17 +105,10 @@ const presets = getAllPresets(); // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. -const tierConfigSummary = (tiers: ComplexityTiers): string => { - const parts = ( - [ - ["Simple", tiers.SIMPLE], - ["Medium", tiers.MEDIUM], - ["Complex", tiers.COMPLEX], - ["Reasoning", tiers.REASONING], - ] as const - ) - .filter(([, models]) => models.length > 0) - .map(([label, models]) => `${label}: ${models.join(", ")}`); +const tierConfigSummary = (config: ComplexityRouterConfigValue): string => { + const parts = activeTierRows(config) + .filter((row) => row.models.length > 0) + .map((row) => `${DEFAULT_TIER_LABELS[row.id as ComplexityTier] ?? activeTierName(row)}: ${row.models.join(", ")}`); return parts.length > 0 ? parts.join(" · ") : "No tiers configured yet"; }; @@ -128,9 +122,9 @@ const getSubmitBlockedReason = ( referencedModelsParams: Parameters[0], availability: ModelAvailability, ): string | null => - getMissingTiersError(config.tiers) ?? + getMissingTiersError(activeTierRows(config)) ?? getTierLabelsError(config.tier_labels) ?? - getPlanModeTierError(config.plan_mode_min_tier, config.tiers) ?? + getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ?? getKeywordTierRulesError(keywordTierRules) ?? getReferencedModelsError(referencedModelsParams, availability); @@ -378,7 +372,7 @@ const AddAutoRouterTab: React.FC = ({ const submitRecommendedRouter = async (name: string) => { const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams; - const missingTiersError = getMissingTiersError(tiers); + const missingTiersError = getMissingTiersError(activeTierRows(complexityRouterConfig)); if (missingTiersError) { setShowValidationErrors(true); toast.fromError(missingTiersError); @@ -423,7 +417,7 @@ const AddAutoRouterTab: React.FC = ({ return; } - const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model); + const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model); const validatedFields = requiresTeamScope ? (["auto_router_name", "team_id"] as const) : (["auto_router_name"] as const); @@ -463,10 +457,12 @@ const AddAutoRouterTab: React.FC = ({ const handleTestConnection = () => { const testTargetParams = { - tiers: complexityRouterConfig.tiers, + tiers: activeTierRows(complexityRouterConfig).map( + (row) => [activeTierName(row), row.models] as [string, string[]], + ), semanticMatchingEnabled, embeddingModel, - defaultModel: resolveComplexityDefaultModel(complexityRouterConfig.tiers, complexityRouterConfig.default_model), + defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model), }; const targets = buildAutoRouterTestTargets(testTargetParams); @@ -581,7 +577,7 @@ const AddAutoRouterTab: React.FC = ({ {!detailsExpanded && ( - {tierConfigSummary(complexityRouterConfig.tiers)} + {tierConfigSummary(complexityRouterConfig)} )} @@ -694,10 +690,7 @@ const AddAutoRouterTab: React.FC = ({ @@ -707,7 +700,6 @@ const AddAutoRouterTab: React.FC = ({ - , ] @@ -744,7 +736,6 @@ const AddAutoRouterTab: React.FC = ({ > Close - , ] diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts index 85fd846ddbc..7c29ea53060 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.test.ts @@ -1,11 +1,18 @@ import { buildAutoRouterTestTargets } from "./build_auto_router_test_targets"; -const tiers = { - SIMPLE: ["gpt-4o-mini"], - MEDIUM: ["claude-sonnet-4"], - COMPLEX: ["claude-sonnet-4"], - REASONING: ["o3"], -}; +const tierEntries = ( + SIMPLE: string[], + MEDIUM: string[] = [], + COMPLEX: string[] = [], + REASONING: string[] = [], +): [string, string[]][] => [ + ["SIMPLE", SIMPLE], + ["MEDIUM", MEDIUM], + ["COMPLEX", COMPLEX], + ["REASONING", REASONING], +]; + +const tiers = tierEntries(["gpt-4o-mini"], ["claude-sonnet-4"], ["claude-sonnet-4"], ["o3"]); describe("buildAutoRouterTestTargets", () => { it("dedups tiers that share a model group into one chat target carrying both labels", () => { @@ -19,7 +26,7 @@ describe("buildAutoRouterTestTargets", () => { it("emits a target per model when a tier has more than one, and dedups across tiers", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini", "claude-sonnet-4"], MEDIUM: ["claude-sonnet-4"], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini", "claude-sonnet-4"], ["claude-sonnet-4"]), semanticMatchingEnabled: false, embeddingModel: undefined, }); @@ -31,7 +38,7 @@ describe("buildAutoRouterTestTargets", () => { it("drops empty/whitespace tiers", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [" "], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"], [], [" "]), semanticMatchingEnabled: false, embeddingModel: undefined, }); @@ -41,7 +48,7 @@ describe("buildAutoRouterTestTargets", () => { it("returns [] when no tier is configured", () => { expect( buildAutoRouterTestTargets({ - tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries([]), semanticMatchingEnabled: false, embeddingModel: undefined, }), @@ -50,7 +57,7 @@ describe("buildAutoRouterTestTargets", () => { it("appends an embedding target only when semantic matching is on and a model is set", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", }); @@ -62,7 +69,7 @@ describe("buildAutoRouterTestTargets", () => { it("omits the embedding target when semantic matching is on but no model is chosen", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: true, embeddingModel: undefined, }); @@ -71,7 +78,7 @@ describe("buildAutoRouterTestTargets", () => { it("omits the embedding target when a model is set but semantic matching is off", () => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: false, embeddingModel: "voyage-3-5", }); @@ -112,7 +119,7 @@ describe("buildAutoRouterTestTargets", () => { it.each([[undefined], [""], [" "]])("adds no default target for %o", (defaultModel) => { const targets = buildAutoRouterTestTargets({ - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tiers: tierEntries(["gpt-4o-mini"]), semanticMatchingEnabled: false, embeddingModel: undefined, defaultModel, diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts index 471552fc84f..70b92dbf8cc 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts @@ -1,5 +1,3 @@ -import { ComplexityTiers } from "./ComplexityRouterConfig"; - export type AutoRouterTestMode = "chat" | "embedding"; export interface AutoRouterTestTarget { @@ -9,7 +7,8 @@ export interface AutoRouterTestTarget { } export interface BuildAutoRouterTestTargetsParams { - tiers: ComplexityTiers; + /** Ordered [tier name, model groups] entries of the active tier set. */ + tiers: readonly (readonly [string, string[]])[]; semanticMatchingEnabled: boolean; embeddingModel: string | undefined; /** The resolved default model - see resolveComplexityDefaultModel. A live fallback destination, @@ -17,23 +16,14 @@ export interface BuildAutoRouterTestTargetsParams { defaultModel?: string; } -// Keys drive iteration order; `satisfies Record` makes it a -// compile error to add a tier to ComplexityTiers without listing it here (and vice versa). -const TIER_ORDER = Object.keys({ - SIMPLE: null, - MEDIUM: null, - COMPLEX: null, - REASONING: null, -} satisfies Record) as (keyof ComplexityTiers)[]; - export const buildAutoRouterTestTargets = ({ tiers, semanticMatchingEnabled, embeddingModel, defaultModel, }: BuildAutoRouterTestTargetsParams): AutoRouterTestTarget[] => { - const tieredByModel = TIER_ORDER.reduce>((acc, tier) => { - return (tiers[tier] ?? []).reduce((tierAcc, rawModel) => { + const tieredByModel = tiers.reduce>((acc, [tier, models]) => { + return models.reduce((tierAcc, rawModel) => { const modelGroup = rawModel?.trim(); if (!modelGroup) return tierAcc; return { ...tierAcc, [modelGroup]: [...(tierAcc[modelGroup] ?? []), tier] }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 33f0fb8a539..63545f5b7de 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -9,6 +9,7 @@ import { hydrateTierLabels, BuildComplexityRouterConfigParams, } from "./build_complexity_router_config"; +import { activeTierRows } from "./tier_rows"; const tiers = { SIMPLE: ["gpt-4o-mini"], @@ -275,30 +276,30 @@ describe("buildComplexityRouterConfig", () => { describe("getMissingTiersError", () => { it("returns null when all four tiers have a model", () => { - expect(getMissingTiersError(tiers)).toBeNull(); + expect(getMissingTiersError(activeTierRows({ tiers: tiers }))).toBeNull(); }); it("names the specific missing tier when only one is blank", () => { - expect(getMissingTiersError({ ...tiers, REASONING: [] })).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, REASONING: [] } }))).toBe( "Select a model for the following tier(s): REASONING", ); }); it("names multiple missing tiers in SIMPLE/MEDIUM/COMPLEX/REASONING order", () => { - expect(getMissingTiersError({ ...tiers, SIMPLE: [], REASONING: [] })).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: [], REASONING: [] } }))).toBe( "Select a model for the following tier(s): SIMPLE, REASONING", ); }); it("names all four tiers when none are filled", () => { const noTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }; - expect(getMissingTiersError(noTiers)).toBe( + expect(getMissingTiersError(activeTierRows({ tiers: noTiers }))).toBe( "Select a model for the following tier(s): SIMPLE, MEDIUM, COMPLEX, REASONING", ); }); it("treats a tier with more than one model as filled", () => { - expect(getMissingTiersError({ ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] })).toBeNull(); + expect(getMissingTiersError(activeTierRows({ tiers: { ...tiers, SIMPLE: ["gpt-4o-mini", "gpt-4o"] } }))).toBeNull(); }); }); @@ -645,15 +646,15 @@ describe("getPlanModeTierError", () => { const tiersWithEmptyComplex = { SIMPLE: ["m1"], MEDIUM: ["m1"], COMPLEX: [], REASONING: [] }; it("passes when the override is off", () => { - expect(getPlanModeTierError(undefined, tiersWithEmptyComplex)).toBeNull(); + expect(getPlanModeTierError(undefined, activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull(); }); it("passes when the named tier has models", () => { - expect(getPlanModeTierError("MEDIUM", tiersWithEmptyComplex)).toBeNull(); + expect(getPlanModeTierError("MEDIUM", activeTierRows({ tiers: tiersWithEmptyComplex }))).toBeNull(); }); it("blocks a tier whose models were removed, which the backend would reject with a 400", () => { - expect(getPlanModeTierError("COMPLEX", tiersWithEmptyComplex)).toContain("COMPLEX"); + expect(getPlanModeTierError("COMPLEX", activeTierRows({ tiers: tiersWithEmptyComplex }))).toContain("COMPLEX"); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index bd95bea226a..241cae25705 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,4 +1,5 @@ import { KeywordTierRule } from "./KeywordTierRules"; +import { type TierRow, activeTierName, tierRowById } from "./tier_rows"; import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; import { TierModelParams, TierModelParamsByTier, serializeTierModelConfigs } from "./complexity_router_tiers"; import { @@ -10,6 +11,7 @@ import { ComplexityTierLabels, ComplexityTiers, DimensionWeights, + TIER_KEYS, TIER_DESCRIPTIONS, TierBoundaries, TokenThresholds, @@ -135,8 +137,6 @@ export interface ComplexityRouterConfigPayload { tier_model_configs?: Record; } -const TIER_KEYS: Array = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; - export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => { const renamed = TIER_KEYS.map((tier) => [tier, tierLabels?.[tier]?.trim() ?? ""] as const).filter( ([tier, label]) => label !== "" && label !== TIER_DESCRIPTIONS[tier].label, @@ -171,26 +171,20 @@ export const getTierLabelsError = (tierLabels: ComplexityTierLabels | undefined) return null; }; -// Requires all 4 tiers non-empty, so the create form can never reach the -// resolveComplexityDefaultModel(tiers, ...) === undefined case — MEDIUM (or SIMPLE) is always -// populated. The edit modal has no equivalent of this check (it allows saving with only some -// tiers filled), which is why it needs its own explicit `!defaultModel` guard after deriving — -// see edit_auto_router_modal.tsx's save handler. A future contributor copying this form's submit -// handler elsewhere should not assume the same guarantee holds without this check. -export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { - const missing = TIER_KEYS.filter((tier) => tiers[tier].length === 0); +// Requires every active tier non-empty, so the create form can never reach the +// resolveComplexityDefaultModel === undefined case. The edit modal allows a partially filled +// set, which is why it keeps its own !defaultModel guard after deriving. +export const getMissingTiersError = (rows: readonly TierRow[]): string | null => { + const missing = rows.filter((row) => row.models.length === 0).map(activeTierName); if (missing.length === 0) return null; return `Select a model for the following tier(s): ${missing.join(", ")}`; }; -// The backend rejects a plan-mode floor naming a tier with no models. The create form's -// getMissingTiersError makes this unreachable there; the edit modal allows partially filled -// tiers, so both gates call this to keep the two forms symmetric. -export const getPlanModeTierError = (planModeMinTier: string | undefined, tiers: ComplexityTiers): string | null => { +export const getPlanModeTierError = (planModeMinTier: string | undefined, rows: readonly TierRow[]): string | null => { if (!planModeMinTier) return null; - const models = tiers[planModeMinTier as keyof ComplexityTiers] ?? []; - if (models.length > 0) return null; - return `The plan-mode minimum tier (${planModeMinTier}) has no models. Add one or turn the override off.`; + const floor = tierRowById(rows, planModeMinTier); + if (floor && floor.models.length > 0) return null; + return `The plan-mode minimum tier (${floor ? activeTierName(floor) : planModeMinTier}) has no models. Add one or turn the override off.`; }; export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => { diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts index 4dffbbd2ac7..be48ff4958d 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts @@ -4,10 +4,10 @@ import { hydrateTierModelParams, normalizeTierModels, pruneTierModelParams, - resolveComplexityDefaultModel, serializeTierModelConfigs, setTierModelReasoningEffort, } from "./complexity_router_tiers"; +import { resolveComplexityDefaultModel } from "./tier_rows"; import type { ComplexityTiers } from "./ComplexityRouterConfig"; @@ -50,31 +50,31 @@ describe("resolveComplexityDefaultModel", () => { const noTiers: ComplexityTiers = { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }; it("derives from MEDIUM first when nothing is pinned", () => { - expect(resolveComplexityDefaultModel(tiers)).toBe("medium-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers })).toBe("medium-model"); }); it("falls back to SIMPLE when MEDIUM is empty", () => { - expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [] })).toBe("simple-model"); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("simple-model"); }); it("derives nothing from COMPLEX or REASONING, which the backend never falls through to", () => { - expect(resolveComplexityDefaultModel({ ...tiers, MEDIUM: [], SIMPLE: [] })).toBeUndefined(); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [], SIMPLE: [] } })).toBeUndefined(); }); it("lets a pin beat the tiers rather than merely filling in for them", () => { - expect(resolveComplexityDefaultModel(tiers, "pinned-model")).toBe("pinned-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers }, "pinned-model")).toBe("pinned-model"); }); it("stands alone as the default when no tier holds a model", () => { - expect(resolveComplexityDefaultModel(noTiers, "pinned-model")).toBe("pinned-model"); + expect(resolveComplexityDefaultModel({ tiers: noTiers }, "pinned-model")).toBe("pinned-model"); }); it.each([[""], [" "], [undefined]])("reads %o as no pin and goes back to the tiers", (pinned) => { - expect(resolveComplexityDefaultModel(tiers, pinned)).toBe("medium-model"); + expect(resolveComplexityDefaultModel({ tiers: tiers }, pinned)).toBe("medium-model"); }); it("resolves to nothing when neither a pin nor a tier offers a model", () => { - expect(resolveComplexityDefaultModel(noTiers)).toBeUndefined(); + expect(resolveComplexityDefaultModel({ tiers: noTiers })).toBeUndefined(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index 2ea1915ca03..ea0d34f6581 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -1,5 +1,5 @@ -import type { ComplexityTiers } from "./ComplexityRouterConfig"; import type { ComplexityTier } from "./KeywordTierRules"; +import { TIER_ORDER } from "./tier_rows"; export type TierModelParams = Record; @@ -80,13 +80,13 @@ export const hydrateTierModelParams = ( * tiers this editor does not render pass through rather than being dropped now the key is managed. */ export const serializeTierModelConfigs = ( - tiers: ComplexityTiers, + tiers: Record, tierModelParams: TierModelParamsByTier | undefined, ): Record | undefined => { if (tierModelParams === undefined) return undefined; const serialized = Object.entries(tierModelParams) .map(([tier, byModel]) => { - const selected = (TIER_ORDER as string[]).includes(tier) ? new Set(tiers[tier as ComplexityTier]) : undefined; + const selected = tier in tiers ? new Set(tiers[tier]) : undefined; const entries = Object.entries(byModel) .filter(([model, params]) => (selected === undefined || selected.has(model)) && Object.keys(params).length > 0) .map(([model_name, litellm_params]) => ({ model_name, litellm_params })); @@ -126,14 +126,6 @@ export const pruneTierModelParams = ( return Object.keys(next).length > 0 ? next : undefined; }; -/** - * Mirrors `init_complexity_router_deployment` (litellm/router.py): an explicit pin wins, otherwise - * the default is `MEDIUM or SIMPLE`. Deriving past SIMPLE would name a model the backend never - * picks, and it raises rather than falling through to COMPLEX/REASONING. - */ -export const resolveComplexityDefaultModel = (tiers: ComplexityTiers, pinned?: string): string | undefined => - pinned?.trim() || tiers.MEDIUM[0] || tiers.SIMPLE[0]; - export const DEFAULT_TIER_LABELS: Record = { SIMPLE: "Simple", MEDIUM: "Medium", @@ -141,8 +133,6 @@ export const DEFAULT_TIER_LABELS: Record = { REASONING: "Reasoning", }; -export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; - export const tierOptions = ( tierLabels: Partial> | undefined, ): { value: ComplexityTier; label: string }[] => diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts new file mode 100644 index 00000000000..54d9e4f3f0a --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { + activeTierName, + activeTierRows, + isBuiltInTierName, + resolveComplexityDefaultModel, + sameTierIdentity, + tierRowById, + tierRowByName, +} from "./tier_rows"; + +const tiers = { SIMPLE: ["a"], MEDIUM: ["b"], COMPLEX: ["c"], REASONING: ["d"] }; + +describe("activeTierRows", () => { + it("reads the tier set as rows whose id is the canonical tier key, in severity order", () => { + expect(activeTierRows({ tiers })).toEqual([ + { id: "SIMPLE", name: "SIMPLE", models: ["a"] }, + { id: "MEDIUM", name: "MEDIUM", models: ["b"] }, + { id: "COMPLEX", name: "COMPLEX", models: ["c"] }, + { id: "REASONING", name: "REASONING", models: ["d"] }, + ]); + }); + + it("gives a tier with no models an empty pool rather than dropping the row", () => { + expect(activeTierRows({ tiers: { ...tiers, COMPLEX: [] } })[2]).toEqual({ + id: "COMPLEX", + name: "COMPLEX", + models: [], + }); + }); + + it("finds a row by id and by name", () => { + const rows = activeTierRows({ tiers }); + expect(tierRowById(rows, "MEDIUM")?.models).toEqual(["b"]); + expect(tierRowById(rows, undefined)).toBeUndefined(); + expect(tierRowByName(rows, " medium ")?.id).toBe("MEDIUM"); + }); +}); + +describe("sameTierIdentity", () => { + it.each([ + ["AUDIT", "audit", true], + ["AUDIT", " audit ", true], + ["AUDIT", "AUDITS", false], + ])("compares %s and %s casefold, matching the backend's uniqueness rule", (left, right, expected) => { + expect(sameTierIdentity(left, right)).toBe(expected); + }); + + it("recognises the four built-in names regardless of case", () => { + expect(["SIMPLE", "medium", "Complex", "REASONING"].every(isBuiltInTierName)).toBe(true); + expect(isBuiltInTierName("SECURITY_REVIEW")).toBe(false); + }); + + it("trims a row name, since the backend matches fallback_tier and keyword rules exactly", () => { + expect(activeTierName({ id: "1", name: " AUDIT ", models: [] })).toBe("AUDIT"); + }); +}); + +describe("resolveComplexityDefaultModel", () => { + it("mirrors init_complexity_router_deployment: a pin wins, then MEDIUM, then SIMPLE", () => { + expect(resolveComplexityDefaultModel({ tiers }, "pinned")).toBe("pinned"); + expect(resolveComplexityDefaultModel({ tiers })).toBe("b"); + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, MEDIUM: [] } })).toBe("a"); + }); + + it("resolves to nothing rather than falling through to COMPLEX, which the backend never picks", () => { + expect(resolveComplexityDefaultModel({ tiers: { ...tiers, SIMPLE: [], MEDIUM: [] } })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts new file mode 100644 index 00000000000..c320980916a --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -0,0 +1,40 @@ +import type { ComplexityTiers } from "./ComplexityRouterConfig"; +import type { ComplexityTier } from "./KeywordTierRules"; + +export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; + +export interface TierRow { + id: string; + name: string; + models: string[]; +} + +export interface ActiveTierSet { + tiers: ComplexityTiers; +} + +export const activeTierName = (row: TierRow): string => row.name.trim(); + +export const sameTierIdentity = (left: string, right: string): boolean => + left.trim().toLowerCase() === right.trim().toLowerCase(); + +export const isBuiltInTierName = (name: string): boolean => TIER_ORDER.some((tier) => sameTierIdentity(tier, name)); + +// The only reader of the tier set. A row's id is the canonical tier key, so anything pointing into +// the set (the plan-mode floor, per-model params) points at a row rather than at a position. +export const activeTierRows = (value: ActiveTierSet): TierRow[] => + TIER_ORDER.map((tier) => ({ id: tier, name: tier, models: value.tiers[tier] ?? [] })); + +export const tierRowById = (rows: readonly TierRow[], id: string | undefined): TierRow | undefined => + id === undefined ? undefined : rows.find((row) => row.id === id); + +export const tierRowByName = (rows: readonly TierRow[], name: string): TierRow | undefined => + rows.find((row) => sameTierIdentity(row.name, name)); + +// Mirrors init_complexity_router_deployment (litellm/router.py): a pin wins, then MEDIUM or SIMPLE +// looked up by exact name. +export const resolveComplexityDefaultModel = (value: ActiveTierSet, pinned?: string): string | undefined => { + const rows = activeTierRows(value); + const named = (name: string) => rows.find((row) => activeTierName(row) === name)?.models[0]; + return pinned?.trim() || named("MEDIUM") || named("SIMPLE"); +}; diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx deleted file mode 100644 index 49a3ae8a001..00000000000 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import { useState } from "react"; -import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; -import { describe, it, expect, vi } from "vitest"; -import DurationSelect from "./DurationSelect"; - -describe("DurationSelect", () => { - it("should render", () => { - render(); - expect(screen.getByRole("combobox")).toBeInTheDocument(); - }); - - it("should render all three duration options", async () => { - const user = userEvent.setup(); - render(); - - const select = screen.getByRole("combobox"); - await user.click(select); - - expect(screen.getByText("Daily")).toBeInTheDocument(); - expect(screen.getByText("Weekly")).toBeInTheDocument(); - expect(screen.getByText("Monthly")).toBeInTheDocument(); - const dailyLabel = screen.getByText("Daily"); - const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; - await user.click(dailyOption); - }); - - it("should apply className prop", () => { - render(); - const select = screen.getByRole("combobox"); - expect(select.closest(".test-class")).toBeInTheDocument(); - }); - - it("should call onChange when an option is selected", async () => { - const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - const onChange = vi.fn(); - render(); - - const select = screen.getByRole("combobox"); - await user.click(select); - - const dailyLabel = screen.getByText("Daily"); - const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; - await user.click(dailyOption); - - expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object)); - }); - - it("should accept and pass value prop to Select", () => { - render(); - const select = screen.getByRole("combobox"); - expect(select).toBeInTheDocument(); - }); - - it.each([ - ["24h", "Daily"], - ["7d", "Weekly"], - ["30d", "Monthly"], - ])("shows the human label on the trigger for %s", (value, label) => { - render(); - - expect(screen.getByRole("combobox")).toHaveTextContent(label); - }); - - it("shows the human label on the trigger after the user picks an option", async () => { - const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - const Harness = () => { - const [value, setValue] = useState("24h"); - return ; - }; - render(); - - await user.click(screen.getByRole("combobox")); - const monthly = screen.getByText("Monthly"); - await user.click(monthly.closest('[role="option"]') ?? monthly); - - expect(screen.getByRole("combobox")).toHaveTextContent("Monthly"); - }); -}); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx deleted file mode 100644 index 55e8aafbae7..00000000000 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; - -interface DurationSelectProps { - className?: string; - value?: string; - onChange?: (value: string, option: { value: string; label: string }) => void; -} - -const DURATION_OPTIONS = [ - { value: "24h", label: "Daily" }, - { value: "7d", label: "Weekly" }, - { value: "30d", label: "Monthly" }, -]; - -export default function DurationSelect({ className, value, onChange }: DurationSelectProps) { - return ( - - ); -} diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index bce96ec76f5..8f4b06d80fb 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -14,25 +14,21 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; -import { - hydrateTierModelParams, - normalizeTierModels, - resolveComplexityDefaultModel, - serializeTierModelConfigs, -} from "../add_model/complexity_router_tiers"; +import { hydrateTierModelParams, normalizeTierModels } from "../add_model/complexity_router_tiers"; +import { type ActiveTierSet, activeTierRows, resolveComplexityDefaultModel } from "../add_model/tier_rows"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; import { + type BuildComplexityRouterConfigParams, + buildComplexityRouterConfig, getKeywordTierRulesError, getSemanticConfigError, getPlanModeTierError, getTierLabelsError, hydrateTierLabels, - normalizeClassifierLlmConfig, - serializeTierLabels, } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching"; -import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model/complexity_router_keywords"; +import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords"; import { hydrateDimensionWeights, hydrateReasoningOverrideMinScore, @@ -46,7 +42,6 @@ import ComplexityRouterConfig, { DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, - heuristicScoringRole, } from "../add_model/ComplexityRouterConfig"; import { Dialog, @@ -119,12 +114,12 @@ const toRecord = (value: unknown): Record => { export const hydratePinnedDefaultModel = ( storedConfigDefaultModel: unknown, litellmParamsDefaultModel: string | null | undefined, - tiers: ComplexityTiers, + activeTiers: ActiveTierSet, ): string | undefined => { if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) { return storedConfigDefaultModel; } - const tierDerived = resolveComplexityDefaultModel(tiers); + const tierDerived = resolveComplexityDefaultModel(activeTiers); const externalOverride = litellmParamsDefaultModel?.trim(); return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined; }; @@ -148,73 +143,48 @@ export const buildUpdatedComplexityRouterConfig = ( if (keywordMatching !== undefined && KEYWORD_MATCHING_KEYS.has(key)) return true; return customTechnicalKeywords !== undefined && key === "custom_technical_keywords"; }; - const preservedConfig = Object.fromEntries(Object.entries(toRecord(storedConfig)).filter(([key]) => !isManaged(key))); - const adaptiveEligible = value.adaptive_eligible ?? "all"; - const storedKeywordRules = keywordMatching ? serializeKeywordTierRules(keywordMatching.keywordTierRules) : []; - const serializedTierLabels = serializeTierLabels(value.tier_labels); - const scorerRuns = heuristicScoringRole(value) !== "never"; - const serializedTierModelConfigs = serializeTierModelConfigs(value.tiers, value.tier_model_params); + const builderParams: BuildComplexityRouterConfigParams = { + tiers: value.tiers, + defaultModel: value.default_model, + planModeMinTier: value.plan_mode_min_tier, + tierLabels: value.tier_labels, + classifierType: value.classifier_type, + classifierLlmConfig: value.classifier_llm_config, + classifierContextWindowSize: value.classifier_context_window_size, + classifierContextBudgetChars: value.classifier_context_budget_chars, + classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, + classifierFallback: value.classifier_fallback, + sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, + deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + customTechnicalKeywords: customTechnicalKeywords ?? [], + keywordTierRules: keywordMatching?.keywordTierRules ?? [], + semanticMatchingEnabled: keywordMatching?.semanticMatchingEnabled ?? false, + embeddingModel: keywordMatching?.embeddingModel, + matchThreshold: keywordMatching?.matchThreshold ?? DEFAULT_MATCH_THRESHOLD, + escalationKeywords: keywordMatching?.escalationKeywords ?? [], + adaptive: value.adaptive ?? false, + adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, + tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, + adaptiveEligible: value.adaptive_eligible ?? "all", + returnRawModelName: value.return_raw_model_name ?? false, + tierBoundaries: value.tier_boundaries, + tokenThresholds: value.token_thresholds, + dimensionWeights: value.dimension_weights, + reasoningOverrideMinScore: value.reasoning_override_min_score, + tierModelParams: value.tier_model_params, + }; + const built = buildComplexityRouterConfig(builderParams); + // Keys this call does not own stay as the stored config left them. + const unowned: readonly string[] = [ + ...(keywordMatching === undefined ? KEYWORD_MATCHING_KEYS : []), + ...(customTechnicalKeywords === undefined ? ["custom_technical_keywords"] : []), + ]; return { ...preservedConfig, - tiers: value.tiers, - ...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }), - ...(value.default_model?.trim() && { default_model: value.default_model }), - ...(value.plan_mode_min_tier?.trim() && { plan_mode_min_tier: value.plan_mode_min_tier }), - ...(serializedTierLabels && { tier_labels: serializedTierLabels }), - classifier_type: value.classifier_type, - ...(value.classifier_type === "llm" && value.classifier_llm_config - ? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) } - : {}), - ...(value.classifier_type === "llm" && - value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }), - ...(value.classifier_type === "llm" && - value.classifier_context_window_size !== undefined && { - classifier_context_window_size: value.classifier_context_window_size, - }), - ...(value.classifier_type === "llm" && - value.classifier_context_budget_chars !== undefined && { - classifier_context_budget_chars: value.classifier_context_budget_chars, - }), - ...(value.classifier_type === "llm" && - value.classifier_context_include_assistant_turns !== undefined && { - classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns, - }), - session_affinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, - deployment_affinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, - ...(customTechnicalKeywords && - customTechnicalKeywords.length > 0 && { - custom_technical_keywords: customTechnicalKeywords, - }), - ...(value.adaptive && { - adaptive: true, - adaptive_weights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, - ...(adaptiveEligible === "all" && { - tier_distance_penalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, - }), - adaptive_eligible: adaptiveEligible, - }), - ...(value.return_raw_model_name && { return_raw_model_name: true }), - ...(keywordMatching && { - // Mirrors buildComplexityRouterConfig: the key only when there is a rule to write, - // escalation keywords always, semantic trio only when on. - ...(storedKeywordRules.length > 0 && { keyword_tier_rules: storedKeywordRules }), - escalation_keywords: keywordMatching.escalationKeywords.map((k) => k.trim()).filter(Boolean), - ...(keywordMatching.semanticMatchingEnabled && { - semantic_keyword_matching: true, - embedding_model: keywordMatching.embeddingModel, - match_threshold: keywordMatching.matchThreshold, - }), - }), - ...(scorerRuns && value.tier_boundaries !== undefined && { tier_boundaries: value.tier_boundaries }), - ...(scorerRuns && value.token_thresholds !== undefined && { token_thresholds: value.token_thresholds }), - ...(scorerRuns && value.dimension_weights !== undefined && { dimension_weights: value.dimension_weights }), - ...(scorerRuns && - value.reasoning_override_min_score !== undefined && { - reasoning_override_min_score: value.reasoning_override_min_score, - }), + ...Object.fromEntries(Object.entries(built).filter(([key]) => !unowned.includes(key))), }; }; @@ -297,7 +267,7 @@ const EditAutoRouterModal: React.FC = ({ ? "Please select at least one model for a complexity tier" : null) ?? getTierLabelsError(complexityRouterConfig.tier_labels) ?? - getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, complexityRouterConfig.tiers) ?? + getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ?? getKeywordTierRulesError(keywordTierRules); useEffect(() => { @@ -355,7 +325,7 @@ const EditAutoRouterModal: React.FC = ({ default_model: hydratePinnedDefaultModel( parsedConfig.default_model, modelData.litellm_params?.complexity_router_default_model, - hydratedTiers, + { tiers: hydratedTiers }, ), plan_mode_min_tier: typeof parsedConfig.plan_mode_min_tier === "string" && parsedConfig.plan_mode_min_tier.trim() !== "" @@ -486,7 +456,7 @@ const EditAutoRouterModal: React.FC = ({ // build_complexity_router_config.ts for why create never can). init_complexity_router_deployment // raises in that case (litellm/router.py), so block it rather than saving a router that // fails at init. - const defaultModel = resolveComplexityDefaultModel(tiers, complexityRouterConfig.default_model); + const defaultModel = resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model); if (!defaultModel) { setShowValidationErrors(true); toast.fromError( diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 65b0a18f78a..f21e98e084c 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -16,7 +16,7 @@ import { stripMaskedSecrets } from "../utils/maskedSecretUtils"; import { truncateString } from "../utils/textUtils"; import AutoRouterConnectionTest from "./add_model/auto_router_connection_test"; import { AutoRouterTestTarget, buildAutoRouterTestTargets } from "./add_model/build_auto_router_test_targets"; -import { normalizeTierModels, resolveComplexityDefaultModel } from "./add_model/complexity_router_tiers"; +import { normalizeTierModels } from "./add_model/complexity_router_tiers"; import { hasAutoRouterEditor, isAutoRouterDeployment, @@ -91,12 +91,10 @@ const buildComplexityRouterTestTargets = ( config = rawConfig; } - const tiers = { - SIMPLE: normalizeTierModels(config.tiers?.SIMPLE), - MEDIUM: normalizeTierModels(config.tiers?.MEDIUM), - COMPLEX: normalizeTierModels(config.tiers?.COMPLEX), - REASONING: normalizeTierModels(config.tiers?.REASONING), - }; + const tiers: [string, string[]][] = + config.tiers && typeof config.tiers === "object" + ? Object.entries(config.tiers).map(([tier, models]) => [tier, normalizeTierModels(models)]) + : []; // Mirrors init_complexity_router_deployment (litellm/router.py): litellm_params wins, otherwise // pure tier-derivation. complexity_router_config.default_model is a UI-only marker the backend @@ -108,7 +106,7 @@ const buildComplexityRouterTestTargets = ( tiers, semanticMatchingEnabled: Boolean(config.semantic_keyword_matching), embeddingModel: config.embedding_model, - defaultModel: resolveComplexityDefaultModel(tiers, effectiveDefaultModel), + defaultModel: effectiveDefaultModel, }; return buildAutoRouterTestTargets(testTargetParams); }; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index c3cc362e29e..d1978ca751f 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -4,7 +4,7 @@ import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils"; -import TeamInfoView from "./TeamInfo"; +import TeamInfoView, { type TeamData } from "./TeamInfo"; const authState = vi.hoisted(() => ({ userRole: "Admin" })); @@ -1613,10 +1613,18 @@ describe("TeamInfoView - which team member fields reach the update payload depen vi.clearAllMocks(); }); - const openEditor = async (user: ReturnType) => { + const openEditor = async ( + user: ReturnType, + teamMemberBudgetTable: TeamData["team_info"]["team_member_budget_table"] = { + max_budget: 42, + budget_duration: "30d", + tpm_limit: 11, + rpm_limit: 22, + }, + ) => { vi.mocked(networking.teamInfoCall).mockResolvedValue( createMockTeamData({ - team_member_budget_table: { max_budget: 42, budget_duration: "30d", tpm_limit: 11, rpm_limit: 22 }, + team_member_budget_table: teamMemberBudgetTable, default_team_member_models: ["gpt-4"], }), ); @@ -1667,6 +1675,46 @@ describe("TeamInfoView - which team member fields reach the update payload depen expect(payload.default_team_member_models).toEqual(["gpt-4"]); }); + it("sends a null team_member_budget_duration when Default Budget Duration is set to never reset", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user); + + await user.click(screen.getByText("Team Member Settings")); + await screen.findByLabelText("Default Budget (USD)"); + await chooseSelectOption(user, screen.getByLabelText("Default Budget Duration"), "Never resets"); + + const payload = await save(user); + + expect(payload.team_member_budget_duration).toBeNull(); + expect(payload.team_member_budget).toBe(42); + expect(JSON.stringify(payload)).toContain('"team_member_budget_duration":null'); + }); + + it("shows Never resets for a stored member budget whose duration is null", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user, { max_budget: 42, budget_duration: null, tpm_limit: null, rpm_limit: null }); + + await user.click(screen.getByText("Team Member Settings")); + + expect(await screen.findByLabelText("Default Budget Duration")).toHaveTextContent("Never resets"); + }); + + it("omits team_member_budget_duration when the dropdown is left untouched on a team with no member budget", async () => { + const user = userEvent.setup({ delay: null }); + await openEditor(user, null); + + await user.click(screen.getByText("Team Member Settings")); + const durationSelect = await screen.findByLabelText("Default Budget Duration"); + expect(durationSelect).toHaveTextContent("Inherit team reset period"); + expect(durationSelect).not.toHaveTextContent("Never resets"); + await user.type(screen.getByLabelText("Default Budget (USD)"), "100"); + + const payload = await save(user); + + expect(payload.team_member_budget).toBe(100); + expect(JSON.parse(JSON.stringify(payload))).not.toHaveProperty("team_member_budget_duration"); + }); + it("omits object_permission.search_tools while Search Tool Settings is closed", async () => { const user = userEvent.setup({ delay: null }); await openEditor(user); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 0b2a2deb72d..6be1054c7fe 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -48,7 +48,7 @@ import { z } from "zod/v4"; import GuardrailsSelect from "./GuardrailsSelect"; import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; -import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; +import BudgetDurationDropdown, { NEVER_RESETS_BUDGET_DURATION } from "../common_components/budget_duration_dropdown"; import { computeTeamModelBadges, normalizeTeamModelSelection, @@ -64,7 +64,6 @@ import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMeta import ModelAliasManager from "../common_components/ModelAliasManager"; import AgentSelector from "../agent_management/AgentSelector"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; -import DurationSelect from "../common_components/DurationSelect"; import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; import GuardrailSettingsView from "../GuardrailSettingsView"; @@ -169,7 +168,7 @@ export interface TeamData { object_permission?: ObjectPermission | null; team_member_budget_table: { max_budget: number; - budget_duration: string; + budget_duration: string | null; tpm_limit: number | null; rpm_limit: number | null; } | null; @@ -1256,7 +1255,15 @@ const TeamInfoView: React.FC = ({ name="team_member_budget_duration" label="Default Budget Duration" > - {({ value, onChange }) => } + {({ id, value, onChange }) => ( + onChange(next === NEVER_RESETS_BUDGET_DURATION ? null : next)} + /> + )} , ): Set => { const { tiers, classifier_llm_config: classifier, embedding_model: embedding, default_model: pinned } = config; - const models = [ - ...tiers.SIMPLE, - ...tiers.MEDIUM, - ...tiers.COMPLEX, - ...tiers.REASONING, - classifier?.model, - embedding, - pinned, - ]; + const models = [...Object.values(tiers).flat(), classifier?.model, embedding, pinned]; // Boolean(), not != null: an empty-string placeholder (e.g. classifier_llm_config seeded before a // model is chosen) is never a real model reference either. return new Set(models.filter((model): model is string => Boolean(model))); @@ -191,7 +182,7 @@ export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability: // effect would block submit for a model that was never going to be submitted. export const getReferencedModelsError = ( params: { - tiers: ComplexityTiers; + tiers: ComplexityRouterConfigPayload["tiers"]; classifierType: ClassifierType; classifierLlmConfig: ClassifierLLMConfig | undefined; semanticMatchingEnabled: boolean; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9af332abd2f..c77b6ad2e46 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11029,11 +11029,9 @@ export interface paths { * -d '{ * "prompt_id": "my_prompt", * "litellm_params": { - * "prompt_id": "json_prompt", + * "prompt_id": "my_prompt", * "prompt_integration": "dotprompt", - * ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED - * "prompt_directory": "/path/to/dotprompt/folder", - * "prompt_data": {"json_prompt": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}}} + * "prompt_data": {"content": "This is a prompt", "metadata": {"model": "gpt-4"}} * }, * "prompt_info": { * "prompt_type": "config"