diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index 37c8efa0ac3..f462421c2c3 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -51,8 +51,7 @@ def is_managed_cloud_storage_uri(file_id: str) -> bool: _S3_SCHEME: Final = "s3://" _FIRST_CHARACTERS_THAT_CAN_DECODE_TO_S3: Final = ("s", "%") -_CLIENT_MEDIA_FIELDS: Final = ("messages", "input", "system") -_MEDIA_URI_KEYS: Final = frozenset({"url", "image_url", "file_id", "file_url", "file_data"}) +_MEDIA_URI_KEYS: Final = frozenset({"url", "image_url", "file_id", "file_url", "file_data", "file_uri", "fileUri"}) def _media_uris(value: object, key: str | None = None) -> Iterator[str]: @@ -96,8 +95,7 @@ def _is_s3_uri(uri: str) -> bool: def s3_media_uris(request_body: Mapping[str, object]) -> tuple[str, ...]: - media_uris: Final = chain.from_iterable(_media_uris(request_body.get(field)) for field in _CLIENT_MEDIA_FIELDS) - return tuple(filter(_is_s3_uri, media_uris)) + return tuple(filter(_is_s3_uri, _media_uris(request_body))) def is_managed_s3_uri(uri: str) -> bool: diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d137876890..61dfc531f0b 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( BEDROCK_MANAGED_S3_UPLOAD_PREFIX, build_managed_cloud_object_name, encode_s3_object_key_for_url, + s3_media_uris, sanitize_cloud_object_component, should_allow_legacy_cloud_file_ids, split_configured_cloud_bucket_name, @@ -391,6 +392,12 @@ def _batch_body_problem(record: Mapping[str, object]) -> str | None: return "has a body that is missing or not a JSON object" if "model" in body and not isinstance(body["model"], str): return "has a body.model that is not a string" + s3_uris: Final = s3_media_uris(body) + if s3_uris: + return ( + f"has s3:// media URLs ({', '.join(s3_uris)}). Bedrock batch files do not take s3:// media URLs yet, " + "so send the media inline as base64 data" + ) return None diff --git a/tests/unit/litellm_core_utils/test_cloud_storage_security.py b/tests/unit/litellm_core_utils/test_cloud_storage_security.py index e1bd97368c9..106f221a886 100644 --- a/tests/unit/litellm_core_utils/test_cloud_storage_security.py +++ b/tests/unit/litellm_core_utils/test_cloud_storage_security.py @@ -71,6 +71,10 @@ def _responses_turn(*parts: object) -> dict[str, object]: return {"input": [{"role": "user", "content": list(parts)}]} +def _gemini_turn(*parts: object) -> dict[str, object]: + return {"contents": [{"role": "user", "parts": list(parts)}]} + + @pytest.mark.parametrize( "request_body", ( @@ -116,6 +120,19 @@ def _responses_turn(*parts: object) -> dict[str, object]: {"system": [{"type": "image", "source": {"type": "url", "url": _MANAGED_UPLOAD}}], "messages": []}, id="anthropic-system", ), + pytest.param( + _gemini_turn({"file_data": {"mime_type": "image/png", "file_uri": _MANAGED_UPLOAD}}), + id="gemini-file-data-snake-case", + ), + pytest.param( + _gemini_turn({"fileData": {"mimeType": "image/png", "fileUri": _MANAGED_UPLOAD}}), + id="gemini-file-data-camel-case", + ), + pytest.param( + {"system_instruction": {"parts": [{"file_data": {"file_uri": _MANAGED_UPLOAD}}]}, "contents": []}, + id="gemini-system-instruction", + ), + pytest.param({"metadata": {"url": _MANAGED_UPLOAD}}, id="media-key-in-any-field"), ), ) def test_s3_media_uris_finds_an_s3_object_on_every_media_surface(request_body: dict[str, object]): @@ -178,7 +195,7 @@ def test_is_managed_s3_uri_leaves_other_objects_alone(uri: str): pytest.param({"messages": [{"role": "user", "content": _MANAGED_UPLOAD}]}, id="string-content"), pytest.param({"input": _MANAGED_UPLOAD}, id="responses-string-input"), pytest.param({"system": _MANAGED_UPLOAD, "messages": []}, id="string-system"), - pytest.param({"metadata": {"url": _MANAGED_UPLOAD}}, id="non-media-field"), + pytest.param({"metadata": {"note": _MANAGED_UPLOAD}}, id="non-media-key"), ), ) def test_s3_media_uris_ignores_what_never_becomes_an_s3_media_source(request_body: dict[str, object]): diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 93bda5b444d..006afc8c492 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -4478,6 +4478,28 @@ def _batch_line_with_body(field: str, value: object) -> str: return _batch_line({**_VALID_BATCH_RECORD["body"], field: value}) +_BATCH_S3_IMAGE: Final = "s3://media-bucket/dir/cat.png" +_BATCH_S3_MEDIA_REFUSAL: Final = ( + f"Batch input file line 2 has s3:// media URLs ({_BATCH_S3_IMAGE}). " + "Bedrock batch files do not take s3:// media URLs yet" +) +_CHAT_BATCH_LINE_WITH_AN_S3_IMAGE: Final = _batch_line_with_body( + "messages", + [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": _BATCH_S3_IMAGE}}]}], +) +_RESPONSES_BATCH_LINE_WITH_AN_S3_IMAGE: Final = json.dumps( + { + "custom_id": "req-2", + "method": "POST", + "url": "/v1/responses", + "body": { + "model": "bedrock/amazon.nova-pro-v1:0", + "input": [{"role": "user", "content": [{"type": "input_image", "image_url": _BATCH_S3_IMAGE}]}], + }, + } +) + + @pytest.mark.usefixtures(httpx_transport.__name__) @pytest.mark.asyncio @pytest.mark.parametrize( @@ -4519,6 +4541,16 @@ def _batch_line_with_body(field: str, value: object) -> str: "Batch input file line 2 could not be converted to a Bedrock batch record", id="claude-list-role-before-a-system-message", ), + pytest.param( + f"{_VALID_BATCH_LINE}\n{_CHAT_BATCH_LINE_WITH_AN_S3_IMAGE}\n".encode(), + _BATCH_S3_MEDIA_REFUSAL, + id="chat-s3-image", + ), + pytest.param( + f"{_VALID_BATCH_LINE}\n{_RESPONSES_BATCH_LINE_WITH_AN_S3_IMAGE}\n".encode(), + _BATCH_S3_MEDIA_REFUSAL, + id="responses-s3-image", + ), pytest.param( f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(), "Batch input file line 2 could not be parsed", diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 605a2b9497f..ca6100f54c1 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -6,7 +6,7 @@ import gzip import json from collections.abc import AsyncIterable, Mapping from types import MappingProxyType, SimpleNamespace -from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence +from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence, TypeAlias from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch @@ -234,15 +234,27 @@ def _anthropic_image(uri: str) -> dict[str, object]: } +def _gemini_snake_case_image(uri: str) -> dict[str, object]: + return {"contents": [{"role": "user", "parts": [{"file_data": {"mime_type": "image/png", "file_uri": uri}}]}]} + + +def _gemini_camel_case_image(uri: str) -> dict[str, object]: + return {"contents": [{"role": "user", "parts": [{"fileData": {"mimeType": "image/png", "fileUri": uri}}]}]} + + +_S3MediaRoute: TypeAlias = Literal["acompletion", "aresponses", "anthropic_messages", "agenerate_content"] +_GEMINI_GENERATE_CONTENT_PATH: Final = "/v1beta/models/nova:generateContent" _S3_MEDIA_SURFACES: Final = ( pytest.param("acompletion", "/v1/chat/completions", _chat_image, id="chat-completions"), pytest.param("aresponses", "/v1/responses", _responses_image, id="responses"), pytest.param("anthropic_messages", "/v1/messages", _anthropic_image, id="anthropic-messages"), + pytest.param("agenerate_content", _GEMINI_GENERATE_CONTENT_PATH, _gemini_snake_case_image, id="gemini-snake-case"), + pytest.param("agenerate_content", _GEMINI_GENERATE_CONTENT_PATH, _gemini_camel_case_image, id="gemini-camel-case"), ) async def _send_s3_media( - route_type: Literal["acompletion", "aresponses", "anthropic_messages"], + route_type: _S3MediaRoute, path: str, body: dict[str, object], user_role: LitellmUserRoles, @@ -596,7 +608,7 @@ class TestProxyBaseLLMRequestProcessing: ) async def test_non_admin_s3_media_is_refused_while_a_proxy_admin_key_passes( self, - route_type: Literal["acompletion", "aresponses", "anthropic_messages"], + route_type: _S3MediaRoute, path: str, body_for: Callable[[str], dict[str, object]], uri: str, @@ -623,13 +635,17 @@ class TestProxyBaseLLMRequestProcessing: ) async def test_non_admin_s3_media_passes_in_a_bucket_the_admin_allowed( self, - route_type: Literal["acompletion", "aresponses", "anthropic_messages"], + route_type: _S3MediaRoute, path: str, body_for: Callable[[str], dict[str, object]], general_settings: dict[str, object], ) -> None: response: Final = await _send_s3_media( - route_type, path, body_for("s3://media-bucket/dir/cat.png"), LitellmUserRoles.INTERNAL_USER, general_settings + route_type, + path, + body_for("s3://media-bucket/dir/cat.png"), + LitellmUserRoles.INTERNAL_USER, + general_settings, ) assert "a cat" in str(response)