From 1a82a13d8bdbd30b1a95604be86277f062f55797 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:12:33 +0000 Subject: [PATCH] fix(router): filter out non-vision deployments when the request contains images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 34 +++++ litellm/router.py | 26 +++- tests/test_litellm/test_router.py | 116 ++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index c43089950ee..f0e9af14311 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -16,6 +16,7 @@ from typing import ( Literal, Mapping, Optional, + Sequence, Tuple, Union, cast, @@ -213,6 +214,39 @@ def _audio_or_image_in_message_content(message: AllMessageValues) -> bool: return False +IMAGE_CONTENT_BLOCK_TYPES: frozenset[str] = frozenset({"image_url", "image", "input_image"}) + + +def _content_blocks_contain_image(content: object) -> bool: + if not isinstance(content, list): + return False + blocks = cast(List[Mapping[str, object]], content) # cast-ok: content blocks are mappings when present + return any(isinstance(block, dict) and block.get("type") in IMAGE_CONTENT_BLOCK_TYPES for block in blocks) + + +def request_contains_image_content( + messages: Optional[Sequence[Mapping[str, object]]] = None, + input: Union[str, Sequence[object], None] = None, +) -> bool: + """ + Whether a request carries image input, across both API surfaces. + + Chat Completions send image blocks inside `messages[*].content` (`image_url`, or + `image` for provider-native shapes); the Responses API sends `input` items that are + either `input_image` blocks themselves or messages holding such blocks. + """ + if messages is not None: + return any(_content_blocks_contain_image(message.get("content")) for message in messages) + if isinstance(input, list): + items = cast(List[Mapping[str, object]], input) # cast-ok: responses input items are mappings + return any( + isinstance(item, dict) + and (item.get("type") in IMAGE_CONTENT_BLOCK_TYPES or _content_blocks_contain_image(item.get("content"))) + for item in items + ) + return False + + def convert_openai_message_to_only_content_messages( messages: List[AllMessageValues], ) -> List[Dict[str, str]]: diff --git a/litellm/router.py b/litellm/router.py index 759d6a0024c..6053b09035f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -32,6 +32,7 @@ from typing import ( List, Literal, Optional, + Sequence, Set, Tuple, TypeVar, @@ -76,6 +77,9 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + request_contains_image_content, +) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, @@ -10189,7 +10193,7 @@ class Router: # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) _returned_deployments = list(healthy_deployments) - invalid_model_indices = set() # Use set for O(1) membership checks + invalid_model_indices: Set[int] = set() # Use set for O(1) membership checks # Token counting (tiktoken) is the dominant on-loop cost for large prompts. # Only count when a deployment actually declares max_input_tokens, and count @@ -10199,11 +10203,16 @@ class Router: _context_window_error = False _potential_error_str = "" _rate_limit_error = False + _vision_error = False parent_otel_span = _get_parent_otel_span_from_kwargs(request_kwargs) raw_instructions = request_kwargs.get("instructions") if request_kwargs else None instructions = raw_instructions if isinstance(raw_instructions, str) else None has_countable_input = messages is not None or input is not None + requires_vision_support = request_contains_image_content( + messages=messages, + input=cast(Union[str, Sequence[object], None], input), # cast-ok: responses input is a str or item list + ) ## get model group RPM ## dt = get_utc_datetime() @@ -10226,6 +10235,11 @@ class Router: model_info = self.get_router_model_info(deployment=deployment, received_model_name=model) _deployment_model = base_model or _litellm_params.get("model", None) + if requires_vision_support and model_info.get("supports_vision") is False: + invalid_model_indices.add(idx) + _vision_error = True + continue + max_input_tokens = model_info.get("max_input_tokens") if isinstance(model_info, dict) else None if isinstance(max_input_tokens, int) and has_countable_input: if input_tokens is None: @@ -10334,6 +10348,16 @@ class Router: model=model, llm_provider="", ) + + elif _vision_error is True: + raise litellm.BadRequestError( + message=( + "litellm._pre_call_checks: No deployments in model group support image input. " + "The request contains image content." + ), + model=model, + llm_provider="", + ) if len(invalid_model_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in invalid_model_indices] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 6db19d63a01..e93d6a8d5e4 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -2959,6 +2959,122 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch): ) +def _vision_router() -> litellm.Router: + """Mixed-modality group: gpt-4o declares supports_vision true in the cost map, + gpt-audio declares it false.""" + return litellm.Router( + model_list=[ + {"model_name": "mixed", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "vision"}}, + {"model_name": "mixed", "litellm_params": {"model": "gpt-audio"}, "model_info": {"id": "text-only"}}, + ], + enable_pre_call_checks=True, + ) + + +IMAGE_MESSAGES = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } +] + + +def test_pre_call_checks_filters_non_vision_deployments_for_image_request(): + """ + An image in the request must eliminate deployments that declare no vision support, + otherwise the provider rejects the whole call and (with session affinity) every + following turn of the session. + """ + router = _vision_router() + + result = router._pre_call_checks( + model="mixed", + healthy_deployments=router.get_model_list(model_name="mixed"), + messages=IMAGE_MESSAGES, + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["vision"] + + +def test_pre_call_checks_keeps_non_vision_deployments_for_text_request(): + """Text-only requests must still be able to land on text-only deployments.""" + router = _vision_router() + + result = router._pre_call_checks( + model="mixed", + healthy_deployments=router.get_model_list(model_name="mixed"), + messages=[{"role": "user", "content": "hello"}], + ) + + assert sorted(deployment["model_info"]["id"] for deployment in result) == ["text-only", "vision"] + + +def test_pre_call_checks_raises_when_no_deployment_supports_images(): + router = litellm.Router( + model_list=[ + {"model_name": "text-group", "litellm_params": {"model": "gpt-audio"}, "model_info": {"id": "d1"}}, + ], + enable_pre_call_checks=True, + ) + + with pytest.raises(litellm.BadRequestError, match="No deployments in model group support image input"): + router._pre_call_checks( + model="text-group", + healthy_deployments=router.get_model_list(model_name="text-group"), + messages=IMAGE_MESSAGES, + ) + + +def test_pre_call_checks_keeps_deployments_with_unknown_vision_support(): + """ + Vision support is unknown (None) for custom/self-hosted models, so those + deployments must not be filtered out; only an explicit `supports_vision: false` + eliminates a deployment. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "custom", + "litellm_params": {"model": "openai/my-private-vlm", "api_base": "http://localhost:8000"}, + "model_info": {"id": "d1"}, + }, + ], + enable_pre_call_checks=True, + ) + + result = router._pre_call_checks( + model="custom", + healthy_deployments=router.get_model_list(model_name="custom"), + messages=IMAGE_MESSAGES, + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["d1"] + + +def test_pre_call_checks_filters_non_vision_deployments_for_responses_image_input(): + """The Responses API sends images as `input_image` items on `input`, not `messages`.""" + router = _vision_router() + + result = router._pre_call_checks( + model="mixed", + healthy_deployments=router.get_model_list(model_name="mixed"), + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is in this image?"}, + {"type": "input_image", "image_url": "https://example.com/cat.png"}, + ], + } + ], + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["vision"] + + def test_count_pre_call_check_tokens_across_api_surfaces(): """ _count_pre_call_check_tokens must count tokens from chat `messages`, a Responses