diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index 0ce25d4b72a..48cc798f631 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -6,6 +6,7 @@ This module handles transforming between: - Responses API format (OpenAI's format with input[], instructions, etc.) """ +import base64 from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Any, Final, cast @@ -25,6 +26,13 @@ from litellm.types.llms.openai import ( _STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"}) +_IMAGE_MAGIC_BYTES: Final[tuple[tuple[bytes, str], ...]] = ( + (b"\x89PNG\r\n\x1a\n", "image/png"), + (b"\xff\xd8\xff", "image/jpeg"), + (b"GIF87a", "image/gif"), + (b"GIF89a", "image/gif"), +) + class LiteLLMResponsesInteractionsConfig: """Configuration class for transforming between Interactions API and Responses API.""" @@ -191,23 +199,37 @@ class LiteLLMResponsesInteractionsConfig: @staticmethod def _transform_image_content_item(item: Mapping[str, object]) -> Mapping[str, object]: - """ - Transform an Interactions API image content part - ({"type": "image", "data": , "mime_type": ...} or {"uri": ...}) - into a Responses API `input_image` part, so it isn't silently dropped. - """ uri: Final = item.get("uri") if isinstance(uri, str) and uri: return {"type": "input_image", "image_url": uri} data: Final = item.get("data") if isinstance(data, str) and data: - mime_type: Final = item.get("mime_type") or "image/jpeg" + mime_type: Final = ( + item.get("mime_type") + or LiteLLMResponsesInteractionsConfig._sniff_image_mime_type(data) + or "application/octet-stream" + ) return {"type": "input_image", "image_url": f"data:{mime_type};base64,{data}"} - # No usable data/uri - fall back to passing through unchanged. return item + @staticmethod + def _sniff_image_mime_type(data: str) -> str | None: + prefix_len = (min(len(data), 24) // 4) * 4 + if prefix_len == 0: + return None + try: + prefix = base64.b64decode(data[:prefix_len]) + except ValueError: + return None + for magic, mime in _IMAGE_MAGIC_BYTES: + if prefix.startswith(magic): + return mime + if prefix[:4] == b"RIFF" and prefix[8:12] == b"WEBP": + return "image/webp" + return None + @staticmethod def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, diff --git a/tests/test_litellm/interactions/test_litellm_responses_bridge.py b/tests/test_litellm/interactions/test_litellm_responses_bridge.py index 26fbd780a95..6244c0b1f09 100644 --- a/tests/test_litellm/interactions/test_litellm_responses_bridge.py +++ b/tests/test_litellm/interactions/test_litellm_responses_bridge.py @@ -5,6 +5,7 @@ Inherits from BaseInteractionsTest to run the same test suite against the litellm_responses bridge provider, which calls litellm.responses() internally. """ +import base64 import os from litellm.interactions.litellm_responses_transformation.transformation import ( @@ -92,10 +93,7 @@ class TestBridgeInputTransformation: } ] - def test_image_content_is_transformed_to_input_image(self): - """Gemini-native image parts must become valid Responses API `input_image` - parts, not pass through unchanged - see GH issue #41427, where the - untransformed part was silently dropped by the Responses API.""" + def test_image_content_with_mime_type_is_transformed_to_input_image(self): image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"} transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( [{"type": "user_input", "content": [image_part]}] @@ -107,6 +105,38 @@ class TestBridgeInputTransformation: } ] + def test_image_content_with_uri_is_transformed_to_input_image(self): + image_part = {"type": "image", "uri": "https://example.com/cat.jpg"} + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "user_input", "content": [image_part]}] + ) + assert transformed == [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "https://example.com/cat.jpg"}], + } + ] + + def test_image_content_missing_mime_type_is_sniffed_from_data(self): + png_signature_b64 = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"rest-of-file").decode() + image_part = {"type": "image", "data": png_signature_b64} + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "user_input", "content": [image_part]}] + ) + assert transformed == [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": f"data:image/png;base64,{png_signature_b64}"}], + } + ] + + def test_image_content_missing_mime_type_and_unrecognized_data_defaults_to_octet_stream(self): + image_part = {"type": "image", "data": "not-a-real-image-signature"} + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "user_input", "content": [image_part]}] + ) + assert transformed[0]["content"][0]["image_url"].startswith("data:application/octet-stream;base64,") + def test_unrecognized_content_type_passes_through_unchanged(self): other_part = {"type": "document", "data": "base64data", "mime_type": "application/pdf"} transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(