fix(proxy): list images in the Codex catalog only for models known to read them (#50738)

* fix(proxy): list images in the Codex catalog only for models known to read them

A translated Codex entry used to offer image input whenever no deployment declared supports_vision,
so Codex let users attach images to models that may not read them. Image input now goes out only
when every deployment behind the name says it reads images.

* test(proxy): type the vision combination test parameters

* refactor(proxy): rename the vision helper and drop the extra vision docstring sentence

litellm/proxy/common_utils/codex_model_translation.py

@@ -95,7 +95,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
     apply_patch_tool_type: Literal["freeform"] | None = None
     truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy()
     experimental_supported_tools: tuple[()] = ()
-    input_modalities: tuple[CodexInputModality, ...] = _TEXT_AND_IMAGE
+    input_modalities: tuple[CodexInputModality, ...] = _TEXT_ONLY
     context_window: int | None
     base_instructions: str
     supports_search_tool: bool = False
@@ -148,8 +148,8 @@ def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDep
 @dataclass(frozen=True, slots=True)
 class CodexCapabilities:
     """What a request to a listed name can count on whichever deployment serves it: a capability is False
-    when one deployment lacks it, True when one has it and none lacks it, None when none says, and the
-    context window is the smallest one any deployment declares."""
+    when one deployment lacks it, True only when none lacks it, None otherwise, and the context window is
+    the smallest one any deployment declares."""
 
     context_window: int | None = None
     supports_vision: bool | None = None
@@ -163,6 +163,13 @@ def _combined(flags: Iterable[bool | None]) -> bool | None:
     return None if not said else False not in said
 
 
+def _declared_by_every_deployment(flags: Iterable[bool | None]) -> bool | None:
+    said: Final = frozenset(flags)
+    if False in said:
+        return False
+    return None if not said or None in said else True
+
+
 def combined_capabilities(
     deployments: Sequence[CodexDeploymentCapabilities], listed_context_window: int | None = None
 ) -> CodexCapabilities:
@@ -175,7 +182,7 @@ def combined_capabilities(
     )
     return CodexCapabilities(
         context_window=min(windows) if windows else listed_context_window,
-        supports_vision=_combined(deployment.supports_vision for deployment in deployments),
+        supports_vision=_declared_by_every_deployment(deployment.supports_vision for deployment in deployments),
         supports_function_calling=_combined(deployment.supports_function_calling for deployment in deployments),
         supports_parallel_function_calling=_combined(
             deployment.supports_parallel_function_calling for deployment in deployments
@@ -230,10 +237,10 @@ def codex_fallback_entry(
 ) -> CodexFallbackModel:
     """Codex's entry for a model it does not know, from the capabilities its deployments share.
 
-    A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform
-    `apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter`
-    is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false
-    it requires.
+    A capability no deployment declares keeps Codex's own default for an unknown model, except image input
+    and the freeform `apply_patch` tool, which go out only to a model known to read images or call tools:
+    `supports_reasoning_summary_parameter` is left out, which Codex reads as true, and the fields only Codex
+    before 0.150.0 reads keep the false it requires.
     """
     levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version)
     return CodexFallbackModel(
@@ -247,7 +254,7 @@ def codex_fallback_entry(
         supports_reasoning_summaries=capabilities.supports_reasoning is True,
         supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True,
         apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None,
-        input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE,
+        input_modalities=_TEXT_AND_IMAGE if capabilities.supports_vision is True else _TEXT_ONLY,
         context_window=capabilities.context_window,
         base_instructions=instructions,
         supports_search_tool=supports_search_tool,

tests/unit/proxy/common_utils/test_codex_model_catalog.py

@@ -1429,20 +1429,25 @@ def _vllm(name: str, host: str, **model_info: object) -> dict[str, object]:
     }
 
 
-def test_a_text_only_deployment_takes_images_off_the_name_and_a_silent_one_does_not() -> None:
+def test_images_are_offered_only_when_every_deployment_behind_the_name_says_it_reads_them() -> None:
     router: Final = litellm.Router(
         model_list=[
             _vllm("mixed", "h0", supports_vision=True),
             _vllm("mixed", "h1", supports_vision=False),
             _vllm("vision-and-silent", "h2", supports_vision=True),
             _vllm("vision-and-silent", "h3"),
+            _vllm("all-vision", "h3a", supports_vision=True),
+            _vllm("all-vision", "h3b", supports_vision=True),
+            _vllm("undeclared", "h3c"),
         ]
     )
 
-    served: Final = _served(router, "mixed", "vision-and-silent")
+    served: Final = _served(router, "mixed", "vision-and-silent", "all-vision", "undeclared")
 
     assert served["mixed"]["input_modalities"] == ["text"]
-    assert served["vision-and-silent"]["input_modalities"] == ["text", "image"]
+    assert served["vision-and-silent"]["input_modalities"] == ["text"]
+    assert served["all-vision"]["input_modalities"] == ["text", "image"]
+    assert served["undeclared"]["input_modalities"] == ["text"]
 
 
 def test_a_silent_deployment_does_not_vote_against_tool_calls_but_one_without_them_does() -> None:

tests/unit/proxy/common_utils/test_codex_model_translation.py

@@ -59,10 +59,8 @@ def test_each_codex_feature_follows_what_the_deployments_declare(flag, field, by
     assert entry.get(field, _ABSENT) == by_value[value]
 
 
-@pytest.mark.parametrize(
-    ("vision", "modalities"), [(False, ["text"]), (True, ["text", "image"]), (None, ["text", "image"])]
-)
-def test_images_are_offered_unless_a_deployment_cannot_read_them(vision, modalities):
+@pytest.mark.parametrize(("vision", "modalities"), [(False, ["text"]), (True, ["text", "image"]), (None, ["text"])])
+def test_images_are_offered_only_when_the_deployments_say_they_read_them(vision, modalities):
     assert _entry(CodexCapabilities(supports_vision=vision))["input_modalities"] == modalities
 
 
@@ -148,7 +146,7 @@ def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_de
     assert entry["truncation_policy"]["mode"] in _CODEX_TRUNCATION_MODES
 
 
-def test_a_deployment_that_says_nothing_does_not_vote():
+def test_a_deployment_that_says_nothing_does_not_vote_on_tools_or_reasoning():
     declares: Final = CodexDeploymentCapabilities(
         supports_vision=True,
         supports_function_calling=True,
@@ -157,7 +155,6 @@ def test_a_deployment_that_says_nothing_does_not_vote():
     )
 
     assert combined_capabilities((declares, _SILENT)) == CodexCapabilities(
-        supports_vision=True,
         supports_function_calling=True,
         supports_parallel_function_calling=True,
         supports_reasoning=True,
@@ -166,6 +163,24 @@ def test_a_deployment_that_says_nothing_does_not_vote():
     assert combined_capabilities(()) == CodexCapabilities()
 
 
+@pytest.mark.parametrize(
+    ("flags", "expected"),
+    [
+        ((True, True), True),
+        ((True, None), None),
+        ((None, None), None),
+        ((), None),
+        ((True, False, None), False),
+    ],
+)
+def test_vision_holds_only_when_every_deployment_says_it_reads_images(
+    flags: tuple[bool | None, ...], expected: bool | None
+) -> None:
+    deployments: Final = tuple(CodexDeploymentCapabilities(supports_vision=flag) for flag in flags)
+
+    assert combined_capabilities(deployments).supports_vision is expected
+
+
 @pytest.mark.parametrize(
     "flag",
     ["supports_vision", "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning"],