mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(interactions): sniff image mime type from data, drop restated comments, add uri test
Defaulting missing mime_type to image/jpeg mislabeled non-JPEG base64 data, so sniff PNG/JPEG/GIF/WEBP magic bytes instead and fall back to application/octet-stream rather than a false guess. Also removes comments that only restated the code, and adds a regression test for the image uri branch.
This commit is contained in:
parent
5f630ed1d2
commit
e0795eac6d
2 changed files with 63 additions and 11 deletions
|
|
@ -6,6 +6,7 @@ This module handles transforming between:
|
|||
- Responses API format (OpenAI's format with input[], instructions, etc.)
|
||||
"""
|
||||
|
||||
import base64
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, cast
|
||||
|
|
@ -25,6 +26,13 @@ from litellm.types.llms.openai import (
|
|||
|
||||
_STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"})
|
||||
|
||||
_IMAGE_MAGIC_BYTES: Final[tuple[tuple[bytes, str], ...]] = (
|
||||
(b"\x89PNG\r\n\x1a\n", "image/png"),
|
||||
(b"\xff\xd8\xff", "image/jpeg"),
|
||||
(b"GIF87a", "image/gif"),
|
||||
(b"GIF89a", "image/gif"),
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMResponsesInteractionsConfig:
|
||||
"""Configuration class for transforming between Interactions API and Responses API."""
|
||||
|
|
@ -191,23 +199,37 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
|
||||
@staticmethod
|
||||
def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
Transform an Interactions API image content part
|
||||
({"type": "image", "data": <base64>, "mime_type": ...} or {"uri": ...})
|
||||
into a Responses API `input_image` part, so it isn't silently dropped.
|
||||
"""
|
||||
uri: Final = item.get("uri")
|
||||
if isinstance(uri, str) and uri:
|
||||
return {"type": "input_image", "image_url": uri}
|
||||
|
||||
data: Final = item.get("data")
|
||||
if isinstance(data, str) and data:
|
||||
mime_type: Final = item.get("mime_type") or "image/jpeg"
|
||||
mime_type: Final = (
|
||||
item.get("mime_type")
|
||||
or LiteLLMResponsesInteractionsConfig._sniff_image_mime_type(data)
|
||||
or "application/octet-stream"
|
||||
)
|
||||
return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"}
|
||||
|
||||
# No usable data/uri - fall back to passing through unchanged.
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def _sniff_image_mime_type(data: str) -> str | None:
|
||||
prefix_len = (min(len(data), 24) // 4) * 4
|
||||
if prefix_len == 0:
|
||||
return None
|
||||
try:
|
||||
prefix = base64.b64decode(data[:prefix_len])
|
||||
except ValueError:
|
||||
return None
|
||||
for magic, mime in _IMAGE_MAGIC_BYTES:
|
||||
if prefix.startswith(magic):
|
||||
return mime
|
||||
if prefix[:4] == b"RIFF" and prefix[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def transform_responses_response_to_interactions_response(
|
||||
responses_response: ResponsesAPIResponse,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Inherits from BaseInteractionsTest to run the same test suite against
|
|||
the litellm_responses bridge provider, which calls litellm.responses() internally.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
|
||||
from litellm.interactions.litellm_responses_transformation.transformation import (
|
||||
|
|
@ -92,10 +93,7 @@ class TestBridgeInputTransformation:
|
|||
}
|
||||
]
|
||||
|
||||
def test_image_content_is_transformed_to_input_image(self):
|
||||
"""Gemini-native image parts must become valid Responses API `input_image`
|
||||
parts, not pass through unchanged - see GH issue #41427, where the
|
||||
untransformed part was silently dropped by the Responses API."""
|
||||
def test_image_content_with_mime_type_is_transformed_to_input_image(self):
|
||||
image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [image_part]}]
|
||||
|
|
@ -107,6 +105,38 @@ class TestBridgeInputTransformation:
|
|||
}
|
||||
]
|
||||
|
||||
def test_image_content_with_uri_is_transformed_to_input_image(self):
|
||||
image_part = {"type": "image", "uri": "https://example.com/cat.jpg"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [image_part]}]
|
||||
)
|
||||
assert transformed == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": "https://example.com/cat.jpg"}],
|
||||
}
|
||||
]
|
||||
|
||||
def test_image_content_missing_mime_type_is_sniffed_from_data(self):
|
||||
png_signature_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"rest-of-file").decode()
|
||||
image_part = {"type": "image", "data": png_signature_b64}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [image_part]}]
|
||||
)
|
||||
assert transformed == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": f"data:image/png;base64,{png_signature_b64}"}],
|
||||
}
|
||||
]
|
||||
|
||||
def test_image_content_missing_mime_type_and_unrecognized_data_defaults_to_octet_stream(self):
|
||||
image_part = {"type": "image", "data": "not-a-real-image-signature"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
[{"type": "user_input", "content": [image_part]}]
|
||||
)
|
||||
assert transformed[0]["content"][0]["image_url"].startswith("data:application/octet-stream;base64,")
|
||||
|
||||
def test_unrecognized_content_type_passes_through_unchanged(self):
|
||||
other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"}
|
||||
transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue