From 5f630ed1d2cf362dfe1818fb182fd30c53cc358d Mon Sep 17 00:00:00 2001 From: Jeevan-threev Date: Thu, 17 Sep 2026 13:33:18 +0530 Subject: [PATCH] fix(interactions): don't silently drop image content in litellm_proxy/Gemini bridge USE_LITELLM_PROXY forces custom_llm_provider="litellm_proxy", which routes interactions.create() through LiteLLMResponsesInteractionsHandler's bridge to litellm.responses(). The bridge's _transform_content_item only recognized {"type": "text"} content parts and passed everything else through unchanged, so Gemini's native image part ({"type": "image", "data": ..., "mime_type": ...}) kept its "image" type tag - not a valid Responses API content type (input_text/input_image/input_file) - and was silently dropped before reaching the model. Add _transform_image_content_item to map Gemini's image part (data+mime_type, or uri) into a proper Responses API input_image part. Fixes #41427 --- .../transformation.py | 24 ++++++++++++++++++- .../test_litellm_responses_bridge.py | 19 +++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index 39ccc26c38c..0ce25d4b72a 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -179,13 +179,35 @@ class LiteLLMResponsesInteractionsConfig: if isinstance(item, str): return {"type": text_type, "text": item} if isinstance(item, Mapping): - if item.get("type") == "text": + item_type = item.get("type") + if item_type == "text": return {"type": text_type, "text": str(item.get("text", ""))} + if item_type == "image": + return LiteLLMResponsesInteractionsConfig._transform_image_content_item(item) return item if isinstance(item, BaseModel): return LiteLLMResponsesInteractionsConfig._transform_content_item(item.model_dump(exclude_none=True), role) return {"type": text_type, "text": str(item)} + @staticmethod + def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]: + """ + Transform an Interactions API image content part + ({"type": "image", "data": , "mime_type": ...} or {"uri": ...}) + into a Responses API `input_image` part, so it isn't silently dropped. + """ + uri: Final = item.get("uri") + if isinstance(uri, str) and uri: + return {"type": "input_image", "image_url": uri} + + data: Final = item.get("data") + if isinstance(data, str) and data: + mime_type: Final = item.get("mime_type") or "image/jpeg" + return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"} + + # No usable data/uri - fall back to passing through unchanged. + return item + @staticmethod def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, diff --git a/tests/test_litellm/interactions/test_litellm_responses_bridge.py b/tests/test_litellm/interactions/test_litellm_responses_bridge.py index 8400f2c4840..26fbd780a95 100644 --- a/tests/test_litellm/interactions/test_litellm_responses_bridge.py +++ b/tests/test_litellm/interactions/test_litellm_responses_bridge.py @@ -92,9 +92,24 @@ class TestBridgeInputTransformation: } ] - def test_non_text_content_passes_through_unchanged(self): + def test_image_content_is_transformed_to_input_image(self): + """Gemini-native image parts must become valid Responses API `input_image` + parts, not pass through unchanged - see GH issue #41427, where the + untransformed part was silently dropped by the Responses API.""" image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"} transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( [{"type": "user_input", "content": [image_part]}] ) - assert transformed == [{"role": "user", "content": [image_part]}] + assert transformed == [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,base64data"}], + } + ] + + def test_unrecognized_content_type_passes_through_unchanged(self): + other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"} + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "user_input", "content": [other_part]}] + ) + assert transformed == [{"role": "user", "content": [other_part]}]