fix(interactions): sniff image mime type from data, drop restated comments, add uri test

Defaulting missing mime_type to image/jpeg mislabeled non-JPEG base64 data,
so sniff PNG/JPEG/GIF/WEBP magic bytes instead and fall back to
application/octet-stream rather than a false guess. Also removes comments that
only restated the code, and adds a regression test for the image uri branch.
This commit is contained in:
Jeevan-threev 2026-09-17 13:43:15 +05:30
parent 5f630ed1d2
commit e0795eac6d
2 changed files with 63 additions and 11 deletions

View file

@ -6,6 +6,7 @@ This module handles transforming between:
- Responses API format (OpenAI's format with input[], instructions, etc.)
"""
import base64
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final, cast
@ -25,6 +26,13 @@ from litellm.types.llms.openai import (
_STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"})
_IMAGE_MAGIC_BYTES: Final[tuple[tuple[bytes, str], ...]] = (
(b"\x89PNG\r\n\x1a\n", "image/png"),
(b"\xff\xd8\xff", "image/jpeg"),
(b"GIF87a", "image/gif"),
(b"GIF89a", "image/gif"),
)
class LiteLLMResponsesInteractionsConfig:
"""Configuration class for transforming between Interactions API and Responses API."""
@ -191,23 +199,37 @@ class LiteLLMResponsesInteractionsConfig:
@staticmethod
def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]:
"""
Transform an Interactions API image content part
({"type": "image", "data": <base64>, "mime_type": ...} or {"uri": ...})
into a Responses API `input_image` part, so it isn't silently dropped.
"""
uri: Final = item.get("uri")
if isinstance(uri, str) and uri:
return {"type": "input_image", "image_url": uri}
data: Final = item.get("data")
if isinstance(data, str) and data:
mime_type: Final = item.get("mime_type") or "image/jpeg"
mime_type: Final = (
item.get("mime_type")
or LiteLLMResponsesInteractionsConfig._sniff_image_mime_type(data)
or "application/octet-stream"
)
return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"}
# No usable data/uri - fall back to passing through unchanged.
return item
@staticmethod
def _sniff_image_mime_type(data: str) -> str | None:
prefix_len = (min(len(data), 24) // 4) * 4
if prefix_len == 0:
return None
try:
prefix = base64.b64decode(data[:prefix_len])
except ValueError:
return None
for magic, mime in _IMAGE_MAGIC_BYTES:
if prefix.startswith(magic):
return mime
if prefix[:4] == b"RIFF" and prefix[8:12] == b"WEBP":
return "image/webp"
return None
@staticmethod
def transform_responses_response_to_interactions_response(
responses_response: ResponsesAPIResponse,

View file

@ -5,6 +5,7 @@ Inherits from BaseInteractionsTest to run the same test suite against
the litellm_responses bridge provider, which calls litellm.responses() internally.
"""
import base64
import os
from litellm.interactions.litellm_responses_transformation.transformation import (
@ -92,10 +93,7 @@ class TestBridgeInputTransformation:
}
]
def test_image_content_is_transformed_to_input_image(self):
"""Gemini-native image parts must become valid Responses API `input_image`
parts, not pass through unchanged - see GH issue #41427, where the
untransformed part was silently dropped by the Responses API."""
def test_image_content_with_mime_type_is_transformed_to_input_image(self):
image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [image_part]}]
@ -107,6 +105,38 @@ class TestBridgeInputTransformation:
}
]
def test_image_content_with_uri_is_transformed_to_input_image(self):
image_part = {"type": "image", "uri": "https://example.com/cat.jpg"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [image_part]}]
)
assert transformed == [
{
"role": "user",
"content": [{"type": "input_image", "image_url": "https://example.com/cat.jpg"}],
}
]
def test_image_content_missing_mime_type_is_sniffed_from_data(self):
png_signature_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"rest-of-file").decode()
image_part = {"type": "image", "data": png_signature_b64}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [image_part]}]
)
assert transformed == [
{
"role": "user",
"content": [{"type": "input_image", "image_url": f"data:image/png;base64,{png_signature_b64}"}],
}
]
def test_image_content_missing_mime_type_and_unrecognized_data_defaults_to_octet_stream(self):
image_part = {"type": "image", "data": "not-a-real-image-signature"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
[{"type": "user_input", "content": [image_part]}]
)
assert transformed[0]["content"][0]["image_url"].startswith("data:application/octet-stream;base64,")
def test_unrecognized_content_type_passes_through_unchanged(self):
other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"}
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(