fix(interactions): don't silently drop image content in litellm_proxy/Gemini bridge

USE_LITELLM_PROXY forces custom_llm_provider="litellm_proxy", which routes
interactions.create() through LiteLLMResponsesInteractionsHandler's bridge to
litellm.responses(). The bridge's _transform_content_item only recognized
{"type": "text"} content parts and passed everything else through unchanged,
so Gemini's native image part ({"type": "image", "data": ..., "mime_type": ...})
kept its "image" type tag - not a valid Responses API content type
(input_text/input_image/input_file) - and was silently dropped before reaching
the model.

Add _transform_image_content_item to map Gemini's image part (data+mime_type,
or uri) into a proper Responses API input_image part.

Fixes #41427
This commit is contained in:
Jeevan-threev 2026-09-17 13:33:18 +05:30
parent 4b368bf066
commit 5f630ed1d2
2 changed files with 40 additions and 3 deletions

View file

@ -179,13 +179,35 @@ class LiteLLMResponsesInteractionsConfig:
if isinstance(item, str):
return {"type": text_type, "text": item}
if isinstance(item, Mapping):
if item.get("type") == "text":
item_type = item.get("type")
if item_type == "text":
return {"type": text_type, "text": str(item.get("text", ""))}
if item_type == "image":
return LiteLLMResponsesInteractionsConfig._transform_image_content_item(item)
return item
if isinstance(item, BaseModel):
return LiteLLMResponsesInteractionsConfig._transform_content_item(item.model_dump(exclude_none=True), role)
return {"type": text_type, "text": str(item)}
@staticmethod
def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]:
"""
Transform an Interactions API image content part
({"type": "image", "data": <base64>, "mime_type": ...} or {"uri": ...})
into a Responses API `input_image` part, so it isn't silently dropped.
"""
uri: Final = item.get("uri")
if isinstance(uri, str) and uri:
return {"type": "input_image", "image_url": uri}
data: Final = item.get("data")
if isinstance(data, str) and data:
mime_type: Final = item.get("mime_type") or "image/jpeg"
return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"}
# No usable data/uri - fall back to passing through unchanged.
return item
@staticmethod
def transform_responses_response_to_interactions_response(
responses_response: ResponsesAPIResponse,

View file

@ -92,9 +92,24 @@ class TestBridgeInputTransformation:
}
]
def test_non_text_content_passes_through_unchanged(self):
def test_image_content_is_transformed_to_input_image(self):
"""Gemini-native image parts must become valid Responses API `input_image`
parts, not pass through unchanged - see GH issue #41427, where the
untransformed part was silently dropped by the Responses API."""
image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [image_part]}]
)
assert transformed == [{"role": "user", "content": [image_part]}]
assert transformed == [
{
"role": "user",
"content": [{"type": "input_image", "image_url": "data:image/png;base64,base64data"}],
}
]
def test_unrecognized_content_type_passes_through_unchanged(self):
other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [other_part]}]
)
assert transformed == [{"role": "user", "content": [other_part]}]