diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index b32f1c71c6a..9deb3fd0ab6 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -951,10 +951,10 @@ def create_anthropic_image_param( # as these providers don't support URL sources for images if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) - return _anthropic_content_element_factory( + return anthropic_content_element_factory( convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) ) - elif format is not None and format.partition(";")[0].strip() == "application/pdf": + elif format is not None and format.partition(";")[0].strip().lower() == "application/pdf": return AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSourceUrl(type="url", url=image_url), @@ -1844,7 +1844,7 @@ def anthropic_document_media_type_for_data_uri(url: str, format_override: str | return None if _data_uri_holds_a_pdf(url): return "application/pdf" - declared: Final = _declared_media_type_for_data_uri(url, format_override) + declared: Final = (_declared_media_type_for_data_uri(url, format_override) or "").lower() return declared if declared in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES else None @@ -2335,16 +2335,16 @@ def _anthropic_native_tool_use_block( return tool_use_param -def _anthropic_content_element_factory( +def anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: - if image_chunk["media_type"] == "application/pdf": + if image_chunk["media_type"].partition(";")[0].strip().lower() == "application/pdf": _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = ( AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type="application/pdf", data=image_chunk["data"], ), ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 7f892cd46d7..882b72fde28 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -42,6 +42,7 @@ def _count_image(block: JsonValue) -> JsonValue: if not isinstance(block, dict) or block.get("type") != "image_url": return block from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_content_element_factory, anthropic_document_block_from_data_uri, anthropic_document_media_type_for_data_uri, convert_to_anthropic_image_obj, @@ -57,8 +58,8 @@ def _count_image(block: JsonValue) -> JsonValue: document: Final = anthropic_document_block_from_data_uri(url, image_block, document_media_type) return {**other_fields, **_COUNT_REQUEST.validate_python(document)} source: Final = convert_to_anthropic_image_obj(openai_image_url=url, format=format_override) - image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source}) - return {**other_fields, **image} + media_block: Final = anthropic_content_element_factory(source) + return {**other_fields, **_COUNT_REQUEST.validate_python(media_block)} def _count_block(block: JsonValue) -> JsonValue: diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index b1ba8c06d7d..a300bd9d161 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -107,10 +107,13 @@ def _user_parts(message: AllMessageValues) -> tuple[object, ...]: return parts_of(message.get("content")) if message["role"] == "user" else () +def _part_type(part: object) -> str: + part_type: Final = message_field(part, "type") + return f"untyped {type(part).__name__}" if part_type is None else str(part_type) + + def _untranslatable_user_part_types(messages: Sequence[AllMessageValues]) -> frozenset[str]: - part_types: Final = frozenset( - str(message_field(part, "type")) for part in chain.from_iterable(map(_user_parts, messages)) - ) + part_types: Final = frozenset(map(_part_type, chain.from_iterable(map(_user_parts, messages)))) return part_types - _CORTEX_CLAUDE_USER_PART_TYPES @@ -243,7 +246,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): drop_params=drop_params, force_base64=True, ) - except (ValueError, BadRequestError) as e: + except (AttributeError, LookupError, TypeError, ValueError, BadRequestError) as e: raise SnowflakeException(status_code=400, message=str(e)) from e return (system or None), conversation diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index d355bfb9d46..4179dafe570 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -5231,8 +5231,12 @@ _LATIN1_TEXT_DATA_URI = "data:text/plain;charset=iso-8859-1;base64," + base64.b6 ({"type": "image_url", "image_url": {"url": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), ({"type": "file", "file": {"file_data": _TEXT_DATA_URI_WITH_CHARSET}}, "the secret code word is MANGO\n"), ({"type": "file", "file": {"file_data": _LATIN1_TEXT_DATA_URI}}, "the secret code word is MANGÓ\n"), + ( + {"type": "image_url", "image_url": {"url": "data:TEXT/PLAIN;base64," + _TEXT_B64}}, + "the secret code word is MANGO\n", + ), ], - ids=["image_url", "file", "image_url_charset", "file_charset", "file_latin1"], + ids=["image_url", "file", "image_url_charset", "file_charset", "file_latin1", "image_url_uppercase"], ) def test_anthropic_messages_pt_text_plain_data_uri_becomes_text_source_document(block, expected_text): messages = [{"role": "user", "content": [{"type": "text", "text": "What is the code word?"}, block]}] @@ -9420,7 +9424,8 @@ def test_anthropic_messages_pt_keeps_a_native_tool_use_answered_by_a_tool_messag ] -def test_create_anthropic_image_param_sends_a_downloaded_pdf_as_a_document(monkeypatch): +@pytest.mark.parametrize("pdf_format", (None, "Application/PDF")) +def test_create_anthropic_image_param_sends_a_downloaded_pdf_as_a_document(monkeypatch, pdf_format): from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache pdf_bytes = b"%PDF-1.4\n1 0 obj\n" @@ -9430,7 +9435,9 @@ def test_create_anthropic_image_param_sends_a_downloaded_pdf_as_a_document(monke monkeypatch.setattr(litellm, "module_level_client", mock_client, raising=False) in_memory_cache.flush_cache() - block = create_anthropic_image_param("https://example.com/report.pdf", is_bedrock_invoke=True) + block = create_anthropic_image_param( + {"url": "https://example.com/report.pdf", "format": pdf_format}, is_bedrock_invoke=True + ) assert block == { "type": "document", @@ -9445,6 +9452,10 @@ def test_create_anthropic_image_param_links_an_https_pdf_as_a_document_only_when "type": "document", "source": {"type": "url", "url": url}, } + assert create_anthropic_image_param({"url": url, "format": "Application/PDF"}) == { + "type": "document", + "source": {"type": "url", "url": url}, + } assert create_anthropic_image_param({"url": url, "format": "image/png"}) == { "type": "image", "source": {"type": "url", "url": url}, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index bd9a3da1357..013a595029c 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -91,6 +91,36 @@ def test_count_sends_a_pdf_image_url_as_the_document_the_chat_request_sends( assert counted["messages"][0]["content"][0]["type"] == "document" +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize( + "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) +) +@pytest.mark.parametrize("pdf_format", (None, "application/pdf")) +def test_count_sends_a_linked_pdf_as_a_document( + config_type: type[AnthropicCountTokensConfig], + pdf_format: str | None, +) -> None: + pdf_bytes: Final = b"%PDF-1.4 linked count fixture" + pdf_url: Final = f"https://1.1.1.1/{config_type.__name__}-{pdf_format}.pdf" + image_url: Final[dict[str, JsonValue]] = {"url": pdf_url} if pdf_format is None else { + "url": pdf_url, "format": pdf_format + } + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": [{"type": "image_url", "image_url": image_url}]} + ] + + with respx.mock: + respx.get(pdf_url).mock( + return_value=httpx.Response(200, content=pdf_bytes, headers={"Content-Type": "application/pdf"}) + ) + counted: Final = config_type().transform_request_to_count_tokens(model="claude-opus-5-5", messages=messages) + + assert counted["messages"][0]["content"][0] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": b64encode(pdf_bytes).decode()}, + } + + @pytest.mark.parametrize( "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) ) diff --git a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py index 0f464b21ed9..849d9db3764 100644 --- a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -760,6 +760,16 @@ class TestSnowflakeCortexClaudeFixes: assert exc_info.value.status_code == 400 assert "input_audio" in str(exc_info.value) + @pytest.mark.parametrize( + ("part", "named"), + (("hello", "untyped str"), ({"type": "image_url", "image_url": {"detail": "low"}}, "url")), + ) + def test_malformed_user_part_is_a_400_naming_it(self, part, named): + with pytest.raises(SnowflakeException) as exc_info: + self._transform([{"role": "user", "content": [part]}]) + assert exc_info.value.status_code == 400 + assert named in str(exc_info.value) + def test_drop_params_drops_a_user_part_cortex_claude_cannot_take(self): body = SnowflakeConfig().transform_request( model="snowflake/claude-sonnet-4-6",