diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b704dfa2675..48f4076d0b5 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -293,15 +293,36 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec return {"type": "image_url", "image_url": {"url": url}} if url else None -def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool: - """Whether any message carries an image content part, across the dialects that reach - pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``, - and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks.""" - return any( +def request_contains_image_content( + messages: Sequence[Mapping[str, object]] | None = None, + input: Sequence[object] | str | None = None, +) -> bool: + """Whether a request carries image input, across both API surfaces. + + Chat Completions carry images in ``messages[*].content`` (``image_url``, Responses + ``input_image``, Anthropic Messages ``image``, including images nested inside + ``tool_result`` blocks); the Responses API carries them in ``input`` items, either as + ``input_image`` blocks or as messages holding such blocks. + """ + if messages is not None and any( isinstance(content, list) and _content_parts_contain_image(content) for message in messages for content in (message.get("content"),) - ) + ): + return True + return isinstance(input, list) and any(_responses_input_item_contains_image(item) for item in input) + + +def _responses_input_item_contains_image(item: object) -> bool: + if not isinstance(item, Mapping): + return False + mapping: Final = cast(Mapping[str, object], item) # cast-ok: Responses input items are string-keyed mappings + if mapping.get("type") in _IMAGE_CONTENT_PART_TYPES: + return True + content: Final = mapping.get("content") + if not isinstance(content, list): + return False + return _content_parts_contain_image(cast(Sequence[object], content)) # cast-ok: content verified as list above def audio_or_image_in_message_content(message: AllMessageValues) -> bool: diff --git a/litellm/router.py b/litellm/router.py index 142b2019e6e..9fd6c0a3aff 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -106,6 +106,9 @@ from litellm.litellm_core_utils.hidden_params import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + request_contains_image_content, +) from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, declares_ptu, @@ -1106,6 +1109,27 @@ def _base_model_widened_window( return MappingProxyType({"max_input_tokens": base_max_input_tokens}) +def _deployment_declares_no_vision(deployment: DeploymentTypedDict) -> bool: + """Whether the deployment's model_info explicitly declares no vision support.""" + if "model_info" not in deployment: + return False + model_info: Final = cast( # cast-ok: model_info is a string-keyed dict, None at runtime + dict[str, object] | None, deployment["model_info"] + ) + if model_info is None: + return False + return model_info.get("supports_vision") is False + + +def _vision_excluded_deployment_indices( + requires_vision_support: bool, deployments: Sequence[DeploymentTypedDict] +) -> frozenset[int]: + """Indices of deployments to skip because the request needs vision support they lack.""" + if not requires_vision_support: + return frozenset() + return frozenset(idx for idx, deployment in enumerate(deployments) if _deployment_declares_no_vision(deployment)) + + class Router: @property def _routing_groups(self) -> dict[str, RoutingGroup]: @@ -13888,6 +13912,19 @@ class Router: if request_kwargs is not None and litellm.drop_params is False else frozenset() ) + requires_vision_support: Final = request_contains_image_content( + messages=messages, + input=cast(Sequence[object] | str | None, input), # cast-ok: forwarded from pre-call args + ) + + ## Vision pre-filter: exclude deployments that declare no vision support when the + ## request carries image input. Computed up front so the per-deployment loop below + ## keeps its existing shape (and complexity budget). + _vision_excluded_indices: Final = _vision_excluded_deployment_indices( + requires_vision_support, + cast(Sequence[DeploymentTypedDict], _returned_deployments), # cast-ok: router deployments are plain dicts + ) + _vision_error: Final = len(_vision_excluded_indices) > 0 ## get model group RPM ## dt: Final = get_utc_datetime() @@ -13967,8 +14004,12 @@ class Router: if no_deployment_takes_params else candidates_lacking_params ) - excluded_indices: Final = frozenset(invalid_model_indices) | frozenset( - idx for (idx, _), fails in zip(param_candidates, candidates_failing_params, strict=True) if fails + excluded_indices: Final = ( + frozenset(invalid_model_indices) + | _vision_excluded_indices + | frozenset( + idx for (idx, _), fails in zip(param_candidates, candidates_failing_params, strict=True) if fails + ) ) if len(excluded_indices) == len(_returned_deployments): @@ -14010,6 +14051,17 @@ class Router: model=model, llm_provider="", ) + + elif _vision_error is True: + raise litellm.BadRequestError( + message=( + f"litellm._pre_call_checks: No deployment of model={model} supports image input. " + "Declare vision support with `model_info: supports_vision: true` on a deployment, " + "or remove the image from the request." + ), + model=model, + llm_provider="", + ) if len(excluded_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices] diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9b1bdbf702e..f6a005cf156 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -28778,3 +28778,131 @@ def test_pre_call_checks_ignores_an_image_response_format_string(monkeypatch: py ) assert [d["model_info"]["id"] for d in filtered] == ["only"] + + +def test_pre_call_checks_drops_deployments_declaring_no_vision_for_image_requests(): + model_list: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "vision", "supports_vision": True}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-audio"}, + "model_info": {"id": "text-only", "supports_vision": False}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "unknown"}, + }, + ] + router: Final = Router(model_list=model_list, enable_pre_call_checks=True) + + filtered: Final = router._pre_call_checks( + model="mixed", + healthy_deployments=model_list, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } + ], + ) + + assert [d["model_info"]["id"] for d in filtered] == ["vision", "unknown"] + + +def test_pre_call_checks_keeps_non_vision_deployments_for_text_requests(): + model_list: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-audio"}, + "model_info": {"id": "text-only", "supports_vision": False}, + }, + ] + router: Final = Router(model_list=model_list, enable_pre_call_checks=True) + + filtered: Final = router._pre_call_checks( + model="mixed", + healthy_deployments=model_list, + messages=[{"role": "user", "content": "hello"}], + ) + + assert [d["model_info"]["id"] for d in filtered] == ["text-only"] + + +def test_pre_call_checks_drops_non_vision_deployments_for_responses_image_input(): + model_list: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "vision", "supports_vision": True}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-audio"}, + "model_info": {"id": "text-only", "supports_vision": False}, + }, + ] + router: Final = Router(model_list=model_list, enable_pre_call_checks=True) + + filtered: Final = router._pre_call_checks( + model="mixed", + healthy_deployments=model_list, + input=[{"type": "input_image", "image_url": "https://example.com/cat.png"}], + ) + + assert [d["model_info"]["id"] for d in filtered] == ["vision"] + + +def test_pre_call_checks_raises_bad_request_when_no_deployment_supports_vision(): + model_list: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "gpt-audio"}, + "model_info": {"id": "text-only", "supports_vision": False}, + }, + ] + router: Final = Router(model_list=model_list, enable_pre_call_checks=True) + + with pytest.raises(litellm.BadRequestError, match="supports image input"): + router._pre_call_checks( + model="mixed", + healthy_deployments=model_list, + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } + ], + ) + + +def test_vision_excluded_deployment_indices(): + from litellm.router import _vision_excluded_deployment_indices + + deployments: Final = [ + {"model_info": {"id": "vision", "supports_vision": True}}, + {"model_info": {"id": "text-only", "supports_vision": False}}, + {"model_info": {"id": "unknown"}}, + ] + + assert _vision_excluded_deployment_indices(True, deployments) == frozenset({1}) + assert _vision_excluded_deployment_indices(False, deployments) == frozenset() + + +def test_deployment_declares_no_vision(): + from litellm.router import _deployment_declares_no_vision + + assert _deployment_declares_no_vision({"model_info": {"supports_vision": False}}) is True + assert _deployment_declares_no_vision({"model_info": {"supports_vision": True}}) is False + assert _deployment_declares_no_vision({"model_info": {}}) is False + assert _deployment_declares_no_vision({"model_info": None}) is False + assert _deployment_declares_no_vision({}) is False