From e09992592385901fef32daca0782962e42fe7b83 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 9 Nov 2026 19:46:37 -0800 Subject: [PATCH] fix(proxy): list images in the Codex catalog only for models known to read them (#50738) * fix(proxy): list images in the Codex catalog only for models known to read them A translated Codex entry used to offer image input whenever no deployment declared supports_vision, so Codex let users attach images to models that may not read them. Image input now goes out only when every deployment behind the name says it reads images. * test(proxy): type the vision combination test parameters * refactor(proxy): rename the vision helper and drop the extra vision docstring sentence --- .../common_utils/codex_model_translation.py | 25 ++++++++++------- .../common_utils/test_codex_model_catalog.py | 11 +++++--- .../test_codex_model_translation.py | 27 ++++++++++++++----- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_translation.py b/litellm/proxy/common_utils/codex_model_translation.py index 694dc422994..49e2615f9fa 100644 --- a/litellm/proxy/common_utils/codex_model_translation.py +++ b/litellm/proxy/common_utils/codex_model_translation.py @@ -95,7 +95,7 @@ class CodexFallbackModel(LiteLLMBaseModel): apply_patch_tool_type: Literal["freeform"] | None = None truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy() experimental_supported_tools: tuple[()] = () - input_modalities: tuple[CodexInputModality, ...] = _TEXT_AND_IMAGE + input_modalities: tuple[CodexInputModality, ...] = _TEXT_ONLY context_window: int | None base_instructions: str supports_search_tool: bool = False @@ -148,8 +148,8 @@ def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDep @dataclass(frozen=True, slots=True) class CodexCapabilities: """What a request to a listed name can count on whichever deployment serves it: a capability is False - when one deployment lacks it, True when one has it and none lacks it, None when none says, and the - context window is the smallest one any deployment declares.""" + when one deployment lacks it, True only when none lacks it, None otherwise, and the context window is + the smallest one any deployment declares.""" context_window: int | None = None supports_vision: bool | None = None @@ -163,6 +163,13 @@ def _combined(flags: Iterable[bool | None]) -> bool | None: return None if not said else False not in said +def _declared_by_every_deployment(flags: Iterable[bool | None]) -> bool | None: + said: Final = frozenset(flags) + if False in said: + return False + return None if not said or None in said else True + + def combined_capabilities( deployments: Sequence[CodexDeploymentCapabilities], listed_context_window: int | None = None ) -> CodexCapabilities: @@ -175,7 +182,7 @@ def combined_capabilities( ) return CodexCapabilities( context_window=min(windows) if windows else listed_context_window, - supports_vision=_combined(deployment.supports_vision for deployment in deployments), + supports_vision=_declared_by_every_deployment(deployment.supports_vision for deployment in deployments), supports_function_calling=_combined(deployment.supports_function_calling for deployment in deployments), supports_parallel_function_calling=_combined( deployment.supports_parallel_function_calling for deployment in deployments @@ -230,10 +237,10 @@ def codex_fallback_entry( ) -> CodexFallbackModel: """Codex's entry for a model it does not know, from the capabilities its deployments share. - A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform - `apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter` - is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false - it requires. + A capability no deployment declares keeps Codex's own default for an unknown model, except image input + and the freeform `apply_patch` tool, which go out only to a model known to read images or call tools: + `supports_reasoning_summary_parameter` is left out, which Codex reads as true, and the fields only Codex + before 0.150.0 reads keep the false it requires. """ levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version) return CodexFallbackModel( @@ -247,7 +254,7 @@ def codex_fallback_entry( supports_reasoning_summaries=capabilities.supports_reasoning is True, supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True, apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None, - input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE, + input_modalities=_TEXT_AND_IMAGE if capabilities.supports_vision is True else _TEXT_ONLY, context_window=capabilities.context_window, base_instructions=instructions, supports_search_tool=supports_search_tool, diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 3db2b90caf7..abdce1a5663 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -1429,20 +1429,25 @@ def _vllm(name: str, host: str, **model_info: object) -> dict[str, object]: } -def test_a_text_only_deployment_takes_images_off_the_name_and_a_silent_one_does_not() -> None: +def test_images_are_offered_only_when_every_deployment_behind_the_name_says_it_reads_them() -> None: router: Final = litellm.Router( model_list=[ _vllm("mixed", "h0", supports_vision=True), _vllm("mixed", "h1", supports_vision=False), _vllm("vision-and-silent", "h2", supports_vision=True), _vllm("vision-and-silent", "h3"), + _vllm("all-vision", "h3a", supports_vision=True), + _vllm("all-vision", "h3b", supports_vision=True), + _vllm("undeclared", "h3c"), ] ) - served: Final = _served(router, "mixed", "vision-and-silent") + served: Final = _served(router, "mixed", "vision-and-silent", "all-vision", "undeclared") assert served["mixed"]["input_modalities"] == ["text"] - assert served["vision-and-silent"]["input_modalities"] == ["text", "image"] + assert served["vision-and-silent"]["input_modalities"] == ["text"] + assert served["all-vision"]["input_modalities"] == ["text", "image"] + assert served["undeclared"]["input_modalities"] == ["text"] def test_a_silent_deployment_does_not_vote_against_tool_calls_but_one_without_them_does() -> None: diff --git a/tests/unit/proxy/common_utils/test_codex_model_translation.py b/tests/unit/proxy/common_utils/test_codex_model_translation.py index 10fdfbd282e..c3b1fd94087 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_translation.py +++ b/tests/unit/proxy/common_utils/test_codex_model_translation.py @@ -59,10 +59,8 @@ def test_each_codex_feature_follows_what_the_deployments_declare(flag, field, by assert entry.get(field, _ABSENT) == by_value[value] -@pytest.mark.parametrize( - ("vision", "modalities"), [(False, ["text"]), (True, ["text", "image"]), (None, ["text", "image"])] -) -def test_images_are_offered_unless_a_deployment_cannot_read_them(vision, modalities): +@pytest.mark.parametrize(("vision", "modalities"), [(False, ["text"]), (True, ["text", "image"]), (None, ["text"])]) +def test_images_are_offered_only_when_the_deployments_say_they_read_them(vision, modalities): assert _entry(CodexCapabilities(supports_vision=vision))["input_modalities"] == modalities @@ -148,7 +146,7 @@ def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_de assert entry["truncation_policy"]["mode"] in _CODEX_TRUNCATION_MODES -def test_a_deployment_that_says_nothing_does_not_vote(): +def test_a_deployment_that_says_nothing_does_not_vote_on_tools_or_reasoning(): declares: Final = CodexDeploymentCapabilities( supports_vision=True, supports_function_calling=True, @@ -157,7 +155,6 @@ def test_a_deployment_that_says_nothing_does_not_vote(): ) assert combined_capabilities((declares, _SILENT)) == CodexCapabilities( - supports_vision=True, supports_function_calling=True, supports_parallel_function_calling=True, supports_reasoning=True, @@ -166,6 +163,24 @@ def test_a_deployment_that_says_nothing_does_not_vote(): assert combined_capabilities(()) == CodexCapabilities() +@pytest.mark.parametrize( + ("flags", "expected"), + [ + ((True, True), True), + ((True, None), None), + ((None, None), None), + ((), None), + ((True, False, None), False), + ], +) +def test_vision_holds_only_when_every_deployment_says_it_reads_images( + flags: tuple[bool | None, ...], expected: bool | None +) -> None: + deployments: Final = tuple(CodexDeploymentCapabilities(supports_vision=flag) for flag in flags) + + assert combined_capabilities(deployments).supports_vision is expected + + @pytest.mark.parametrize( "flag", ["supports_vision", "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning"],