fix(router): filter out non-vision deployments when the request contains images

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -16,6 +16,7 @@ from typing import (
     Literal,
     Mapping,
     Optional,
+    Sequence,
     Tuple,
     Union,
     cast,
@@ -213,6 +214,39 @@ def _audio_or_image_in_message_content(message: AllMessageValues) -> bool:
     return False
 
 
+IMAGE_CONTENT_BLOCK_TYPES: frozenset[str] = frozenset({"image_url", "image", "input_image"})
+
+
+def _content_blocks_contain_image(content: object) -> bool:
+    if not isinstance(content, list):
+        return False
+    blocks = cast(List[Mapping[str, object]], content)  # cast-ok: content blocks are mappings when present
+    return any(isinstance(block, dict) and block.get("type") in IMAGE_CONTENT_BLOCK_TYPES for block in blocks)
+
+
+def request_contains_image_content(
+    messages: Optional[Sequence[Mapping[str, object]]] = None,
+    input: Union[str, Sequence[object], None] = None,
+) -> bool:
+    """
+    Whether a request carries image input, across both API surfaces.
+
+    Chat Completions send image blocks inside `messages[*].content` (`image_url`, or
+    `image` for provider-native shapes); the Responses API sends `input` items that are
+    either `input_image` blocks themselves or messages holding such blocks.
+    """
+    if messages is not None:
+        return any(_content_blocks_contain_image(message.get("content")) for message in messages)
+    if isinstance(input, list):
+        items = cast(List[Mapping[str, object]], input)  # cast-ok: responses input items are mappings
+        return any(
+            isinstance(item, dict)
+            and (item.get("type") in IMAGE_CONTENT_BLOCK_TYPES or _content_blocks_contain_image(item.get("content")))
+            for item in items
+        )
+    return False
+
+
 def convert_openai_message_to_only_content_messages(
     messages: List[AllMessageValues],
 ) -> List[Dict[str, str]]:

litellm/router.py

@@ -32,6 +32,7 @@ from typing import (
     List,
     Literal,
     Optional,
+    Sequence,
     Set,
     Tuple,
     TypeVar,
@@ -76,6 +77,9 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    request_contains_image_content,
+)
 from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.litellm_core_utils.sensitive_data_masker import (
     SensitiveDataMasker,
@@ -10189,7 +10193,7 @@ class Router:
         # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
         _returned_deployments = list(healthy_deployments)
 
-        invalid_model_indices = set()  # Use set for O(1) membership checks
+        invalid_model_indices: Set[int] = set()  # Use set for O(1) membership checks
 
         # Token counting (tiktoken) is the dominant on-loop cost for large prompts.
         # Only count when a deployment actually declares max_input_tokens, and count
@@ -10199,11 +10203,16 @@ class Router:
         _context_window_error = False
         _potential_error_str = ""
         _rate_limit_error = False
+        _vision_error = False
         parent_otel_span = _get_parent_otel_span_from_kwargs(request_kwargs)
 
         raw_instructions = request_kwargs.get("instructions") if request_kwargs else None
         instructions = raw_instructions if isinstance(raw_instructions, str) else None
         has_countable_input = messages is not None or input is not None
+        requires_vision_support = request_contains_image_content(
+            messages=messages,
+            input=cast(Union[str, Sequence[object], None], input),  # cast-ok: responses input is a str or item list
+        )
 
         ## get model group RPM ##
         dt = get_utc_datetime()
@@ -10226,6 +10235,11 @@ class Router:
                 model_info = self.get_router_model_info(deployment=deployment, received_model_name=model)
                 _deployment_model = base_model or _litellm_params.get("model", None)
 
+                if requires_vision_support and model_info.get("supports_vision") is False:
+                    invalid_model_indices.add(idx)
+                    _vision_error = True
+                    continue
+
                 max_input_tokens = model_info.get("max_input_tokens") if isinstance(model_info, dict) else None
                 if isinstance(max_input_tokens, int) and has_countable_input:
                     if input_tokens is None:
@@ -10334,6 +10348,16 @@ class Router:
                     model=model,
                     llm_provider="",
                 )
+
+            elif _vision_error is True:
+                raise litellm.BadRequestError(
+                    message=(
+                        "litellm._pre_call_checks: No deployments in model group support image input. "
+                        "The request contains image content."
+                    ),
+                    model=model,
+                    llm_provider="",
+                )
         if len(invalid_model_indices) > 0:
             # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
             _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in invalid_model_indices]

tests/test_litellm/test_router.py

@@ -2959,6 +2959,122 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch):
         )
 
 
+def _vision_router() -> litellm.Router:
+    """Mixed-modality group: gpt-4o declares supports_vision true in the cost map,
+    gpt-audio declares it false."""
+    return litellm.Router(
+        model_list=[
+            {"model_name": "mixed", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "vision"}},
+            {"model_name": "mixed", "litellm_params": {"model": "gpt-audio"}, "model_info": {"id": "text-only"}},
+        ],
+        enable_pre_call_checks=True,
+    )
+
+
+IMAGE_MESSAGES = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "what is in this image?"},
+            {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
+        ],
+    }
+]
+
+
+def test_pre_call_checks_filters_non_vision_deployments_for_image_request():
+    """
+    An image in the request must eliminate deployments that declare no vision support,
+    otherwise the provider rejects the whole call and (with session affinity) every
+    following turn of the session.
+    """
+    router = _vision_router()
+
+    result = router._pre_call_checks(
+        model="mixed",
+        healthy_deployments=router.get_model_list(model_name="mixed"),
+        messages=IMAGE_MESSAGES,
+    )
+
+    assert [deployment["model_info"]["id"] for deployment in result] == ["vision"]
+
+
+def test_pre_call_checks_keeps_non_vision_deployments_for_text_request():
+    """Text-only requests must still be able to land on text-only deployments."""
+    router = _vision_router()
+
+    result = router._pre_call_checks(
+        model="mixed",
+        healthy_deployments=router.get_model_list(model_name="mixed"),
+        messages=[{"role": "user", "content": "hello"}],
+    )
+
+    assert sorted(deployment["model_info"]["id"] for deployment in result) == ["text-only", "vision"]
+
+
+def test_pre_call_checks_raises_when_no_deployment_supports_images():
+    router = litellm.Router(
+        model_list=[
+            {"model_name": "text-group", "litellm_params": {"model": "gpt-audio"}, "model_info": {"id": "d1"}},
+        ],
+        enable_pre_call_checks=True,
+    )
+
+    with pytest.raises(litellm.BadRequestError, match="No deployments in model group support image input"):
+        router._pre_call_checks(
+            model="text-group",
+            healthy_deployments=router.get_model_list(model_name="text-group"),
+            messages=IMAGE_MESSAGES,
+        )
+
+
+def test_pre_call_checks_keeps_deployments_with_unknown_vision_support():
+    """
+    Vision support is unknown (None) for custom/self-hosted models, so those
+    deployments must not be filtered out; only an explicit `supports_vision: false`
+    eliminates a deployment.
+    """
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "custom",
+                "litellm_params": {"model": "openai/my-private-vlm", "api_base": "http://localhost:8000"},
+                "model_info": {"id": "d1"},
+            },
+        ],
+        enable_pre_call_checks=True,
+    )
+
+    result = router._pre_call_checks(
+        model="custom",
+        healthy_deployments=router.get_model_list(model_name="custom"),
+        messages=IMAGE_MESSAGES,
+    )
+
+    assert [deployment["model_info"]["id"] for deployment in result] == ["d1"]
+
+
+def test_pre_call_checks_filters_non_vision_deployments_for_responses_image_input():
+    """The Responses API sends images as `input_image` items on `input`, not `messages`."""
+    router = _vision_router()
+
+    result = router._pre_call_checks(
+        model="mixed",
+        healthy_deployments=router.get_model_list(model_name="mixed"),
+        input=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": "what is in this image?"},
+                    {"type": "input_image", "image_url": "https://example.com/cat.png"},
+                ],
+            }
+        ],
+    )
+
+    assert [deployment["model_info"]["id"] for deployment in result] == ["vision"]
+
+
 def test_count_pre_call_check_tokens_across_api_surfaces():
     """
     _count_pre_call_check_tokens must count tokens from chat `messages`, a Responses