mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(interactions): don't silently drop image content in litellm_proxy/Gemini bridge
USE_LITELLM_PROXY forces custom_llm_provider="litellm_proxy", which routes
interactions.create() through LiteLLMResponsesInteractionsHandler's bridge to
litellm.responses(). The bridge's _transform_content_item only recognized
{"type": "text"} content parts and passed everything else through unchanged,
so Gemini's native image part ({"type": "image", "data": ..., "mime_type": ...})
kept its "image" type tag - not a valid Responses API content type
(input_text/input_image/input_file) - and was silently dropped before reaching
the model.
Add _transform_image_content_item to map Gemini's image part (data+mime_type,
or uri) into a proper Responses API input_image part.
Fixes #41427
This commit is contained in:
parent
4b368bf066
commit
5f630ed1d2
2 changed files with 40 additions and 3 deletions
|
|
@ -179,13 +179,35 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
if isinstance(item, str):
|
||||
return {"type": text_type, "text": item}
|
||||
if isinstance(item, Mapping):
|
||||
if item.get("type") == "text":
|
||||
item_type = item.get("type")
|
||||
if item_type == "text":
|
||||
return {"type": text_type, "text": str(item.get("text", ""))}
|
||||
if item_type == "image":
|
||||
return LiteLLMResponsesInteractionsConfig._transform_image_content_item(item)
|
||||
return item
|
||||
if isinstance(item, BaseModel):
|
||||
return LiteLLMResponsesInteractionsConfig._transform_content_item(item.model_dump(exclude_none=True), role)
|
||||
return {"type": text_type, "text": str(item)}
|
||||
|
||||
@staticmethod
|
||||
def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
Transform an Interactions API image content part
|
||||
({"type": "image", "data": <base64>, "mime_type": ...} or {"uri": ...})
|
||||
into a Responses API `input_image` part, so it isn't silently dropped.
|
||||
"""
|
||||
uri: Final = item.get("uri")
|
||||
if isinstance(uri, str) and uri:
|
||||
return {"type": "input_image", "image_url": uri}
|
||||
|
||||
data: Final = item.get("data")
|
||||
if isinstance(data, str) and data:
|
||||
mime_type: Final = item.get("mime_type") or "image/jpeg"
|
||||
return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"}
|
||||
|
||||
# No usable data/uri - fall back to passing through unchanged.
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def transform_responses_response_to_interactions_response(
|
||||
responses_response: ResponsesAPIResponse,
|
||||
|
|
|
|||
|
|
@ -92,9 +92,24 @@ class TestBridgeInputTransformation:
|
|||
}
|
||||
]
|
||||
|
||||
def test_non_text_content_passes_through_unchanged(self):
|
||||
def test_image_content_is_transformed_to_input_image(self):
|
||||
"""Gemini-native image parts must become valid Responses API `input_image`
|
||||
parts, not pass through unchanged - see GH issue #41427, where the
|
||||
untransformed part was silently dropped by the Responses API."""
|
||||
image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [image_part]}]
|
||||
)
|
||||
assert transformed == [{"role": "user", "content": [image_part]}]
|
||||
assert transformed == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": "data:image/png;base64,base64data"}],
|
||||
}
|
||||
]
|
||||
|
||||
def test_unrecognized_content_type_passes_through_unchanged(self):
|
||||
other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [other_part]}]
|
||||
)
|
||||
assert transformed == [{"role": "user", "content": [other_part]}]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue