diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index fa6f1c1ed0a..a5e4cb396b7 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,7 +5,7 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from functools import partial @@ -531,6 +531,29 @@ async def async_convert_urls_to_base64( ) +async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError": + try: + return await async_convert_url_to_base64(url, timeout=timeout) + except litellm.ImageFetchError as error: + return error + + +async def _downloads_through_first_failure( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None +) -> "AsyncIterator[str | litellm.ImageFetchError]": + for url in remote_urls: + yield (outcome := await _download_outcome(url, timeout)) + if isinstance(outcome, litellm.ImageFetchError): + return + + +async def async_fetch_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> "Mapping[str, str | litellm.ImageFetchError]": + outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)]) + return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) + + def _remote_urls_to_inline( messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] ) -> tuple[str, ...]: @@ -539,6 +562,41 @@ def _remote_urls_to_inline( return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote)))) +def _message_parts(messages: Iterable[AllMessageValues]) -> Iterator[object]: + return chain.from_iterable(_content_parts(message) for message in messages) + + +def _file_of_file_part(part: object) -> Mapping[str, object] | None: + fields: Final = _as_mapping(part) + return _as_mapping(fields.get("file")) if fields is not None and fields.get("type") == "file" else None + + +def _file_id_url(part: object, url_starts: tuple[str, ...]) -> str | None: + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file_id if isinstance(file_id, str) and file_id.startswith(url_starts) else None + + +def _is_readable_file_part_or_other_part(part: object) -> bool: + fields: Final = _as_mapping(part) + if fields is None or fields.get("type") != "file": + return fields is not None + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file is not None and (not file_id or isinstance(file_id, str)) + + +def has_file_id_urls(messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]) -> bool: + return any(_file_id_url(part, url_starts) is not None for part in _message_parts(messages)) + + +def file_id_urls_before_an_unreadable_part( + messages: Iterable[AllMessageValues], url_starts: tuple[str, ...] +) -> tuple[str, ...]: + readable_parts: Final = takewhile(_is_readable_file_part_or_other_part, _message_parts(messages)) + return tuple(dict.fromkeys(url for part in readable_parts if (url := _file_id_url(part, url_starts)) is not None)) + + def inline_remote_media( messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues] should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url, diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 9c222f03c8c..872e153c91a 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -34,6 +34,9 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None: class DeepSeekChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list[str]: """ DeepSeek reasoner models support thinking parameter. diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py index 68b63812248..bd8e335b617 100644 --- a/litellm/llms/docker_model_runner/chat/transformation.py +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -25,6 +25,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig): The engine name (e.g., "llama.cpp") is part of the API endpoint path. """ + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index 002078dd04b..e9cfb5e09cf 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -22,6 +22,9 @@ class HerokuError(Exception): class HerokuChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index d13e46658ae..1dfb98fc65a 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -142,8 +142,6 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ - _is_base_class = False - temperature: int | None = None top_p: int | None = None max_tokens: int | None = None diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 044c0131699..f7bd46536a1 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -2,9 +2,11 @@ Support for gpt model family """ +import asyncio import json import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from contextvars import ContextVar from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -28,7 +30,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, + async_fetch_urls_to_base64, convert_url_to_base64, + file_id_urls_before_an_unreadable_part, + has_file_id_urls, ) from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( drop_non_python_regex_patterns, @@ -85,6 +90,20 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) +_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool") +_PDF_URL_STARTS: Final = ("https://", "http://", "www.") +_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) +_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( + "prefetched_pdf_urls" +) + + +def _pdf_data_url(url: str) -> str: + prefetched: Final = _prefetched_pdf_urls.get(_NOTHING_PREFETCHED).get(url) + if isinstance(prefetched, litellm.ImageFetchError): + raise prefetched + return prefetched if prefetched is not None else convert_url_to_base64(url) + def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: if custom_llm_provider != "openai": @@ -191,9 +210,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. """ - # Add a class variable to track if this is the base class - _is_base_class = True - frequency_penalty: int | None = None function_call: str | dict | None = None functions: list | None = None @@ -225,8 +241,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) - self.__class__._is_base_class = False - @classmethod def get_config(cls): return super().get_config() @@ -322,9 +336,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool: - potential_pdf_url_starts: Final = ["https://", "http://", "www."] file_id: Final = content_item.get("file_id") - if file_id and any(file_id.startswith(start) for start in potential_pdf_url_starts): + if file_id and any(file_id.startswith(start) for start in _PDF_URL_STARTS): return True return False @@ -332,7 +345,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): content_copy: Final = content_item.copy() file_id: Final = content_copy.get("file_id") if file_id is not None: - base64_data: Final = convert_url_to_base64(file_id) + base64_data: Final = _pdf_data_url(file_id) content_copy["file_data"] = base64_data content_copy["filename"] = "my_file.pdf" content_copy.pop("file_id") @@ -688,6 +701,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): **self._sanitized_tools_update_for_openai(optional_params, litellm_params), } + def _flattens_content_lists(self) -> bool: + return False + + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + return messages + async def async_transform_request( self, model: str, @@ -696,30 +715,25 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect( - model=model, - messages=await self._transform_messages( - messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True - ), - tools=optional_params.get("tools", []), - prompt_cache_options=_prompt_cache_options(optional_params), - litellm_params=litellm_params, - request_params=optional_params, - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools - if self.__class__._is_base_class: - if not optional_params.get("tools") and not optional_params.get("functions"): - optional_params.pop("tool_choice", None) - return { - "model": model, - "messages": transformed_messages, - **optional_params, - **self._sanitized_tools_update_for_openai(optional_params, litellm_params), - } - else: - ## allow for any object specific behaviour to be handled + if not has_file_id_urls(messages, _PDF_URL_STARTS): return self.transform_request(model, messages, optional_params, litellm_params, headers) + sent_messages: Final = self._with_provider_roles( + self._prompt_cache_ordered_messages(messages, litellm_params), model + ) + pdf_urls: Final = file_id_urls_before_an_unreadable_part( + (message for message in sent_messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED), + _PDF_URL_STARTS, + ) + prefetched: Final = ( + _NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls) + ) + reset_token: Final = _prefetched_pdf_urls.set(prefetched) + try: + return await asyncio.to_thread( + self.transform_request, model, messages, optional_params, litellm_params, headers + ) + finally: + _prefetched_pdf_urls.reset(reset_token) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 4d56a3b38c5..06257af7075 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -122,6 +122,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): return super()._map_openai_params(non_default_params, optional_params, model, drop_params) + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + keeps_system_messages: Final = supports_system_messages(model, "openai") + return tuple( + message + if keeps_system_messages or message["role"] != "system" + else ChatCompletionUserMessage(content=message["content"], role="user") + for message in messages + ) + def is_model_o_series_model(self, model: str) -> bool: model = model.split("/")[-1] # could be "openai/o3" or "o3" return ( @@ -152,13 +161,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - modalities: image => drop param (if user opts in to dropping param) - role: system ==> translate to role 'user' """ - _supports_system_messages: Final = supports_system_messages(model, "openai") - transformed_messages: Final = list(messages) - for i, message in enumerate(transformed_messages): - if message["role"] == "system" and not _supports_system_messages: - new_message = ChatCompletionUserMessage(content=message["content"], role="user") - transformed_messages[i] = new_message # Replace the old message with the new one - + transformed_messages: Final = list(self._with_provider_roles(messages, model)) if is_async: return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index f330d7fc6d4..495ebf594d8 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1166,7 +1166,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): shared_session: Optional["ClientSession"] = None, ): response = None - data = provider_config.transform_request( + data = await provider_config.async_transform_request( model=model, messages=messages, optional_params=optional_params, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 66b41967ca7..05cf90d7abd 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -50,6 +50,9 @@ def create_config_class(provider: SimpleProviderConfig): base_class: Final[type] = OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig class JSONProviderConfig(base_class): + def _flattens_content_lists(self) -> bool: + return bool(provider.special_handling.get("convert_content_list_to_string")) + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] @@ -69,7 +72,7 @@ def create_config_class(provider: SimpleProviderConfig): """Transform messages based on special_handling config""" # Handle content list to string conversion if configured - if provider.special_handling.get("convert_content_list_to_string"): + if self._flattens_content_lists(): messages = handle_messages_with_content_list_to_str_conversion(messages) if is_async: diff --git a/litellm/llms/sambanova/chat.py b/litellm/llms/sambanova/chat.py index 0350950f342..8247ec2770d 100644 --- a/litellm/llms/sambanova/chat.py +++ b/litellm/llms/sambanova/chat.py @@ -32,6 +32,9 @@ class SambanovaConfig(OpenAIGPTConfig): response_format: dict | None = None tools: list | None = None + def _flattens_content_lists(self) -> bool: + return True + def __init__( self, max_tokens: int | None = None, diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 263c2f7618d..964b39551a2 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1,15 +1,18 @@ """ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import asyncio import base64 import copy import json +import threading from collections.abc import Callable from typing import Final +import httpx import pytest import respx -from httpx import Response +from httpx import Request, Response import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -17,11 +20,21 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_MEDIA_BOUNDARY, ) from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache +from litellm.llms.bedrock.chat.chat_completions.transformation import AmazonBedrockRuntimeChatCompletionsConfig +from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig +from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig +from litellm.llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig +from litellm.llms.heroku.chat.transformation import HerokuChatConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import SimpleProviderConfig +from litellm.llms.sambanova.chat import SambanovaConfig +from litellm.llms.vertex_ai.vertex_gemma_models.transformation import VertexGemmaConfig class TestOpenAIGPTConfig: @@ -1348,6 +1361,607 @@ async def test_openai_media_download_preserves_supported_chat_fields( +_PDF_BYTES: Final = b"%PDF-1.7\nmock-pdf" +_DOWNLOADED_PDF_PART: Final = { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{base64.b64encode(_PDF_BYTES).decode('utf-8')}", + "filename": "my_file.pdf", + }, +} + + +def _pdf_url_part(url: str) -> dict[str, object]: + return {"type": "file", "file": {"file_id": url}} + + +async def _serve_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _tool_result_messages(*content: object) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "read the report"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(content)}, + ] + + +@pytest.fixture +def pdf_urls_reachable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +_SUMMARIZE: Final = {"type": "text", "text": "Summarize the report."} +_UPLOADED_FILE_PART: Final = {"type": "file", "file": {"file_id": "file-uploaded"}} +_ONCE_URL: Final = "https://files.example.test/download-once.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "expected_messages"), + [ + pytest.param( + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _DOWNLOADED_PDF_PART]}], + id="user_message", + ), + pytest.param( + _tool_result_messages(_SUMMARIZE, _pdf_url_part(_ONCE_URL)), + [ + *_tool_result_messages(_SUMMARIZE)[:2], + {"role": "tool", "tool_call_id": "call_1", "content": [_SUMMARIZE]}, + {"role": "user", "content": [{"type": "text", "text": TOOL_RESULT_MEDIA_BOUNDARY}, _DOWNLOADED_PDF_PART]}, + ], + id="tool_result", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_url_once_with_the_async_client( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], expected_messages: list[dict[str, object]] +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == expected_messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cache_once( + respx_mock: respx.MockRouter, +) -> None: + pdf_url: Final = "https://files.example.test/too-large-to-cache.pdf" + large_pdf: Final = _PDF_BYTES + b"0" * in_memory_cache.max_size_per_item * 1024 + + async def serve_large_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=large_pdf) + + download: Final = respx_mock.get(pdf_url).mock(side_effect=serve_large_pdf_to_async_clients_only) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"][0]["content"][1]["file"] == { + "file_data": f"data:application/pdf;base64,{base64.b64encode(large_pdf).decode('utf-8')}", + "filename": "my_file.pdf", + } + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_pdf_urls_one_at_a_time_for_a_host_that_refuses_bursts( + respx_mock: respx.MockRouter, +) -> None: + pdf_urls: Final = tuple(f"https://files.example.test/in-turn-{index}.pdf" for index in range(3)) + for pdf_url in pdf_urls: + in_memory_cache.delete_cache(pdf_url) + download_running: Final = asyncio.Lock() + + async def refuse_a_second_download_at_once(request: Request) -> Response: + if download_running.locked(): + return Response(429) + async with download_running: + await asyncio.sleep(0.05) + return await _serve_pdf_to_async_clients_only(request) + + downloads: Final = respx_mock.route(method="GET", host="files.example.test").mock( + side_effect=refuse_a_second_download_at_once + ) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, *map(_pdf_url_part, pdf_urls)]}], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}] + assert downloads.call_count == len(pdf_urls) + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(respx_mock: respx.MockRouter) -> None: + pdf_url: Final = "https://files.example.test/streamed.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + chunk: Final = { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "seen"}, "finish_reason": "stop"}], + } + chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode(), + ) + ) + + stream: Final = await litellm.acompletion( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + api_key="sk-openai-test", + stream=True, + ) + text: Final = "".join([part.choices[0].delta.content or "" async for part in stream]) + + assert text == "seen" + assert download.call_count == 1 + assert json.loads(chat.calls[0].request.content)["messages"] == [ + {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]} + ] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("unreadable_part", "error"), + [ + pytest.param( + {"type": "file", "file": None}, + "Content block has type='file' but is missing the required 'file' field", + id="file_part_without_its_file", + ), + pytest.param( + {"type": "file", "file": "file-uploaded"}, "has no attribute 'items'", id="file_that_is_not_an_object" + ), + pytest.param( + {"type": "file", "file": {"file_id": 123}}, "has no attribute 'startswith'", id="file_id_not_a_string" + ), + pytest.param("Summarize the report.", "has no attribute 'get'", id="part_that_is_not_an_object"), + ], +) +async def test_async_transform_request_rejects_an_unreadable_part_like_the_sync_transform_before_a_later_pdf_url( + respx_mock: respx.MockRouter, unreadable_part: object, error: str +) -> None: + pdf_url: Final = "https://files.example.test/after-a-broken-part.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(return_value=Response(200, content=b"%PDF-1.4")) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, unreadable_part, _pdf_url_part(pdf_url)]}] + + with pytest.raises((litellm.BadRequestError, AttributeError), match=error) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + with pytest.raises(type(sync_error.value), match=error): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert not download.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"]) +async def test_async_transform_request_sends_a_plain_string_tool_result_part_like_the_sync_transform( + config: OpenAIGPTConfig, +) -> None: + messages: Final = _tool_result_messages("plain string result") + + request: Final = await config.async_transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request == config.transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + +def _json_provider_config(special_handling: dict[str, bool]) -> OpenAIGPTConfig: + provider: Final = SimpleProviderConfig( + "test-json-provider", + { + "base_url": "https://json-provider.example.test/v1", + "api_key_env": "TEST_JSON_PROVIDER_API_KEY", + "special_handling": special_handling, + }, + ) + return create_config_class(provider)() + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config", + [ + pytest.param(SambanovaConfig(), id="sambanova"), + pytest.param(HerokuChatConfig(), id="heroku"), + pytest.param(DockerModelRunnerChatConfig(), id="docker_model_runner"), + pytest.param(_json_provider_config({"convert_content_list_to_string": True}), id="json_provider"), + pytest.param(DeepSeekChatConfig(), id="deepseek"), + ], +) +async def test_async_transform_request_does_not_download_a_pdf_url_for_a_provider_that_sends_only_text( + respx_mock: respx.MockRouter, config: OpenAIGPTConfig +) -> None: + pdf_url: Final = "https://files.example.test/dropped.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await config.async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 0 + assert request["messages"] == [{"role": "user", "content": "Summarize the report."}] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "messages", "expected_messages"), + [ + pytest.param( + HerokuChatConfig(), + [{"role": "user", "content": [_pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}], + id="pdf_only_message_for_a_provider_that_sends_only_text", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_the_sync_transform_reaches_off_the_event_loop( + respx_mock: respx.MockRouter, + config: OpenAIGPTConfig, + messages: list[dict[str, object]], + expected_messages: list[dict[str, object]], +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + event_loop_thread: Final = threading.get_ident() + + def serve_pdf_off_the_event_loop_only(request: Request) -> Response: + if threading.get_ident() == event_loop_thread: + return Response(503) + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + respx_mock.get(_ONCE_URL).mock(side_effect=serve_pdf_off_the_event_loop_only) + + request: Final = await config.async_transform_request( + model="test-model-without-system-messages", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == expected_messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_system_pdf_with_the_async_client_for_a_model_without_system_messages( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[{"role": "system", "content": [_pdf_url_part(_ONCE_URL)]}, {"role": "user", "content": [_SUMMARIZE]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == [ + {"role": "user", "content": [_DOWNLOADED_PDF_PART]}, + {"role": "user", "content": [_SUMMARIZE]}, + ] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_url_with_the_async_client_for_a_json_provider_that_keeps_lists( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await _json_provider_config({}).async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + +def _serve_pdf(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _answer_not_found(request: Request) -> Response: + return Response(404) + + +def _answer_too_many_requests(request: Request) -> Response: + return Response(429) + + +def _refuse_connection(request: Request) -> Response: + raise httpx.ConnectError("connection refused", request=request) + + +_PARITY_URL: Final = "https://files.example.test/parity.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_PARITY_URL)]}], id="user_message"), + pytest.param( + [ + { + "role": "user", + "content": [_SUMMARIZE, {"type": "file", "file": {"file_id": _PARITY_URL, "filename": "report.pdf"}}], + } + ], + id="named_file", + ), + pytest.param(_tool_result_messages(_SUMMARIZE, _pdf_url_part(_PARITY_URL)), id="tool_result"), + pytest.param( + [ + {"role": "system", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": [_SUMMARIZE]}, + {"role": "assistant", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": "Go on."}, + ], + id="system_and_assistant_messages", + ), + ], +) +@pytest.mark.parametrize( + ("config", "model"), + [ + pytest.param(OpenAIGPTConfig(), "gpt-4.1", id="openai"), + pytest.param(AmazonBedrockRuntimeChatCompletionsConfig(), "us.xai.grok-4.6", id="bedrock_chat_completions"), + pytest.param(VertexGemmaConfig(), "gemma-3-27b-it", id="vertex_gemma"), + ], +) +async def test_async_transform_request_matches_the_sync_request_downloads_and_callers_messages( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], config: OpenAIGPTConfig, model: str +) -> None: + download: Final = respx_mock.get(_PARITY_URL).mock(side_effect=_serve_pdf) + sync_messages: Final = copy.deepcopy(messages) + async_messages: Final = copy.deepcopy(messages) + + in_memory_cache.delete_cache(_PARITY_URL) + sync_request: Final = config.transform_request( + model=model, messages=sync_messages, optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = download.call_count + in_memory_cache.delete_cache(_PARITY_URL) + async_request: Final = await config.async_transform_request( + model=model, messages=async_messages, optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request + assert async_messages == sync_messages + assert download.call_count - sync_downloads == sync_downloads + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dead_url_answer", + [ + pytest.param(_answer_not_found, id="not_found"), + pytest.param(_answer_too_many_requests, id="rate_limited"), + pytest.param(_refuse_connection, id="refused"), + ], +) +async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count( + respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response] +) -> None: + healthy_url: Final = "https://files.example.test/healthy.pdf" + dead_url: Final = "https://files.example.test/dead.pdf" + healthy: Final = respx_mock.get(healthy_url).mock(side_effect=_serve_pdf) + dead: Final = respx_mock.get(dead_url).mock(side_effect=dead_url_answer) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(healthy_url), _pdf_url_part(dead_url)]}] + + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = healthy.call_count + dead.call_count + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as async_error: + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert str(async_error.value) == str(sync_error.value) + assert healthy.call_count + dead.call_count - sync_downloads == sync_downloads + + +_DEAD_URL: Final = "https://files.example.test/dead-first.pdf" +_AFTER_DEAD_URL: Final = "https://files.example.test/after-dead.pdf" +_WWW_FILE_ID: Final = "www.files.example.test/no-scheme.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "model", "messages"), + [ + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_DEAD_URL), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="dead_url", + ), + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_WWW_FILE_ID), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="www_file_id", + ), + pytest.param( + OpenAIOSeriesConfig(), + "test-model-without-system-messages", + [ + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + ], + id="system_message_for_a_model_without_system_messages", + ), + ], +) +async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it( + respx_mock: respx.MockRouter, config: OpenAIGPTConfig, model: str, messages: list[dict[str, object]] +) -> None: + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + with pytest.raises(litellm.ImageFetchError): + await config.async_transform_request( + model=model, messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) + + assert later.call_count == 0 + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_moved_system_pdf_first_when_system_messages_go_first( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + with pytest.raises(litellm.ImageFetchError): + await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[ + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + ], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert later.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", ["system", "assistant", "developer"]) +@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")]) +async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request( + role: str, part_type: object +) -> None: + messages: Final = [ + {"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]}, + {"role": "user", "content": "hi"}, + ] + + sync_request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + async_request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_a_pdf_url_that_failed_in_an_async_request_downloads_again_in_a_later_request_of_the_same_task( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=[Response(404), _serve_pdf(Request("GET", _ONCE_URL))]) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}] + + with pytest.raises(litellm.ImageFetchError): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" @@ -1560,14 +2174,8 @@ class TestSystemMessagesFirst: @pytest.mark.asyncio async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): - class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): - _is_base_class = True - - def __init__(self) -> None: - pass - monkeypatch.setattr(litellm, "openai_system_messages_first", True) - request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + request = await self.config.async_transform_request( model="gpt-4.1", messages=self._messages(), optional_params={},