From cd64d3252fc91b38cfdaf6b9bc58121c6b1767ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:35:53 -0700 Subject: [PATCH 1/8] fix(openai): transform async chat messages once, whatever configs were built before --- .../llms/openai/chat/gpt_transformation.py | 42 ++++----- .../chat/test_openai_gpt_transformation.py | 88 +++++++++++++++++-- 2 files changed, 95 insertions(+), 35 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..e64033e143b 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -4,7 +4,9 @@ Support for gpt model family import json import os +from asyncio import to_thread from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -136,6 +138,14 @@ def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | Non return extra_body.get("prompt_cache_options") +def _attached_files(message: AllMessageValues) -> Iterator[ChatCompletionFileObjectFile]: + match message: + case {"role": "user", "content": list() as content}: + return (part["file"] for part in content if part.get("type") == "file" and "file" in part) + case _: + return iter(()) + + class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): """ Reference: https://platform.openai.com/docs/api-reference/chat/create @@ -163,9 +173,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. """ - # Add a class variable to track if this is the base class - _is_base_class = True - frequency_penalty: int | None = None function_call: str | dict | None = None functions: list | None = None @@ -197,8 +204,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) - self.__class__._is_base_class = False - @classmethod def get_config(cls): return super().get_config() @@ -646,29 +651,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect( - model=model, - messages=await self._transform_messages( - messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True - ), - tools=optional_params.get("tools", []), - prompt_cache_options=_prompt_cache_options(optional_params), - litellm_params=litellm_params, - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools - if self.__class__._is_base_class: - if not optional_params.get("tools") and not optional_params.get("functions"): - optional_params.pop("tool_choice", None) - return { - "model": model, - "messages": transformed_messages, - **optional_params, - **self._sanitized_tools_update_for_openai(optional_params, litellm_params), - } - else: - ## allow for any object specific behaviour to be handled + if not self._downloads_pdf_url(messages): return self.transform_request(model, messages, optional_params, litellm_params, headers) + return await to_thread(self.transform_request, model, messages, optional_params, litellm_params, headers) + + def _downloads_pdf_url(self, messages: Sequence[AllMessageValues]) -> bool: + return any(self.contains_pdf_url(file) for file in chain.from_iterable(map(_attached_files, messages))) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..2b93aada819 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1,10 +1,12 @@ """ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import asyncio import base64 import copy import json -from collections.abc import Callable +import threading +from collections.abc import Callable, Coroutine from typing import Final import pytest @@ -1348,6 +1350,82 @@ async def test_openai_media_download_preserves_supported_chat_fields( +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_codestral_file_request_still_works_after_an_openai_config_is_built( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chat: Final = respx_mock.post("https://codestral.example.test/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + OpenAIGPTConfig() + + await litellm.acompletion( + model="codestral/codestral-latest", + api_base="https://codestral.example.test/v1", + api_key="mistral-test-key", + messages=[{"role": "user", "content": [ + {"type": "text", "text": "What is in the file?"}, + {"type": "file", "file": {"file_id": "uploaded-file-id"}}, + ]}], + ) + + assert json.loads(chat.calls[0].request.content)["messages"][0]["content"][1] == { + "type": "file", "file_id": "uploaded-file-id", + } + + +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + pdf_url: Final = "https://files.example.test/off-event-loop.pdf" + pdf_content: Final = b"%PDF-1.7\nmock-pdf" + in_memory_cache.delete_cache(pdf_url) + event_loop_thread: Final = threading.current_thread() + event_loop_ran: Final = threading.Event() + loop_ran_during_download: Final[list[bool]] = [] + + def pdf_response() -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=pdf_content) + + async def serve_pdf_on_event_loop() -> Response: + loop_ran_during_download.append(await asyncio.to_thread(event_loop_ran.wait, 5)) + return pdf_response() + + def serve_pdf(request: object) -> Response | Coroutine[object, object, Response]: + if threading.current_thread() is event_loop_thread: + return serve_pdf_on_event_loop() + loop_ran_during_download.append(event_loop_ran.wait(timeout=5)) + return pdf_response() + + respx_mock.get(pdf_url).mock(side_effect=serve_pdf) + + async def mark_event_loop_ran() -> None: + event_loop_ran.set() + + request, _ = await asyncio.gather( + OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": pdf_url}}]}], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ), + mark_event_loop_ran(), + ) + + assert loop_ran_during_download == [True] + assert request["messages"][0]["content"][0]["file"] == { + "file_data": f"data:application/pdf;base64,{base64.b64encode(pdf_content).decode('utf-8')}", + "filename": "my_file.pdf", + } + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" @@ -1560,14 +1638,8 @@ class TestSystemMessagesFirst: @pytest.mark.asyncio async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): - class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): - _is_base_class = True - - def __init__(self) -> None: - pass - monkeypatch.setattr(litellm, "openai_system_messages_first", True) - request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + request = await self.config.async_transform_request( model="gpt-4.1", messages=self._messages(), optional_params={}, From acb9f0abb15b648e92b2e685614c0bdcc2521e00 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:13:35 -0700 Subject: [PATCH 2/8] fix(openai): keep tool-result PDF downloads off the event loop and stop Mistral rewriting retried messages --- litellm/llms/mistral/chat/transformation.py | 32 +++++++++-------- .../llms/openai/chat/gpt_transformation.py | 11 ++++-- .../test_mistral_chat_transformation.py | 28 +++++++++++++++ .../chat/test_openai_gpt_transformation.py | 36 +++++++++++++++---- 4 files changed, 84 insertions(+), 23 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 2fb9402ed03..8b14ae126ad 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -63,6 +63,23 @@ def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a de return isinstance(value, list) +def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + match block: + case {"type": "file", "file": {"file_id": file_id}} if file_id: + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + case _: + return block + + +def _with_mistral_file_ids(message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + return cast( # cast-ok: dict spread keeps message shape + AllMessageValues, {**message, "content": [_with_mistral_file_id(block) for block in content]} + ) + + class MistralConfig(OpenAIGPTConfig): """ Reference: https://docs.mistral.ai/api/ @@ -363,20 +380,7 @@ class MistralConfig(OpenAIGPTConfig): """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [_with_mistral_file_ids(message) for message in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index e64033e143b..ad3f12a1486 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -4,8 +4,11 @@ Support for gpt model family import json import os -from asyncio import to_thread +from asyncio import get_running_loop from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from functools import partial from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload @@ -81,6 +84,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) +_PDF_URL_TRANSFORM_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-pdf-url-transform") def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: @@ -140,7 +144,7 @@ def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | Non def _attached_files(message: AllMessageValues) -> Iterator[ChatCompletionFileObjectFile]: match message: - case {"role": "user", "content": list() as content}: + case {"role": "user" | "tool", "content": list() as content}: return (part["file"] for part in content if part.get("type") == "file" and "file" in part) case _: return iter(()) @@ -653,7 +657,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) -> dict: if not self._downloads_pdf_url(messages): return self.transform_request(model, messages, optional_params, litellm_params, headers) - return await to_thread(self.transform_request, model, messages, optional_params, litellm_params, headers) + transform: Final = partial(self.transform_request, model, messages, optional_params, litellm_params, headers) + return await get_running_loop().run_in_executor(_PDF_URL_TRANSFORM_EXECUTOR, copy_context().run, transform) def _downloads_pdf_url(self, messages: Sequence[AllMessageValues]) -> bool: return any(self.contains_pdf_url(file) for file in chain.from_iterable(map(_attached_files, messages))) diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b76e1707805..13683ee6a78 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -799,6 +799,34 @@ class TestMistralFileHandling: assert result[0]["content"][2]["file_id"] == "file-67890" # type: ignore +@pytest.mark.asyncio +async def test_mistral_sends_an_uploaded_file_again_when_the_same_messages_are_retried(): + messages: Final = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize the attached file."}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + ) + config: Final = MistralConfig() + + requests: Final = [ + await config.async_transform_request( + model="codestral-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + for _ in range(2) + ] + + assert [request["messages"][0]["content"][1] for request in requests] == [ + {"type": "file", "file_id": "file-12345"} + ] * 2 + + class TestMistralStripsOutputOnlyFields: """Mistral rejects unknown input fields with a 422 ``extra_forbidden``. diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 2b93aada819..9ab4ca66ba8 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.types.llms.openai import AllMessageValues class TestOpenAIGPTConfig: @@ -1376,16 +1377,39 @@ async def test_codestral_file_request_still_works_after_an_openai_config_is_buil } +_PDF_URL: Final = "https://files.example.test/off-event-loop.pdf" +_PDF_URL_PART: Final = {"type": "file", "file": {"file_id": _PDF_URL}} + + @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "user", "content": [_PDF_URL_PART]}], id="user_message"), + pytest.param( + [ + {"role": "user", "content": "read the report"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [_PDF_URL_PART]}, + ], + id="tool_result", + ), + ], +) async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_event_loop( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, messages: list[AllMessageValues] ) -> None: monkeypatch.setattr(litellm, "user_url_validation", False) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - pdf_url: Final = "https://files.example.test/off-event-loop.pdf" pdf_content: Final = b"%PDF-1.7\nmock-pdf" - in_memory_cache.delete_cache(pdf_url) + in_memory_cache.delete_cache(_PDF_URL) event_loop_thread: Final = threading.current_thread() event_loop_ran: Final = threading.Event() loop_ran_during_download: Final[list[bool]] = [] @@ -1403,7 +1427,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e loop_ran_during_download.append(event_loop_ran.wait(timeout=5)) return pdf_response() - respx_mock.get(pdf_url).mock(side_effect=serve_pdf) + respx_mock.get(_PDF_URL).mock(side_effect=serve_pdf) async def mark_event_loop_ran() -> None: event_loop_ran.set() @@ -1411,7 +1435,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e request, _ = await asyncio.gather( OpenAIGPTConfig().async_transform_request( model="gpt-4.1", - messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": pdf_url}}]}], + messages=copy.deepcopy(messages), optional_params={}, litellm_params={"custom_llm_provider": "openai"}, headers={}, @@ -1420,7 +1444,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e ) assert loop_ran_during_download == [True] - assert request["messages"][0]["content"][0]["file"] == { + assert request["messages"][-1]["content"][-1]["file"] == { "file_data": f"data:application/pdf;base64,{base64.b64encode(pdf_content).decode('utf-8')}", "filename": "my_file.pdf", } From 025811569175c24f10fef5aeefb7f0ef0634ec32 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:04:48 -0700 Subject: [PATCH 3/8] fix(openai): pre-download PDF URLs into the sync transform so async requests match sync Async chat requests now download every PDF URL in user and tool messages at once, then run the sync transform_request on a worker thread that reads those downloads. The request, filename, logged messages, and download count match a sync request, and a failed download raises the sync error with no second attempt. DeepSeek joins the configs that flatten content and skip the up-front download. --- .../prompt_templates/image_handling.py | 21 ++ litellm/llms/deepseek/chat/transformation.py | 3 + .../chat/transformation.py | 3 + litellm/llms/heroku/chat/transformation.py | 3 + .../llms/openai/chat/gpt_transformation.py | 37 ++- litellm/llms/openai_like/dynamic_config.py | 5 +- litellm/llms/sambanova/chat.py | 3 + .../chat/test_openai_gpt_transformation.py | 259 ++++++++++++++++-- 8 files changed, 307 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index f458661241f..41a5f586429 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -326,6 +326,23 @@ async def async_convert_urls_to_base64( raise +async def _fetch_outcome( + url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None +) -> "str | litellm.ImageFetchError": + try: + return await _fetch_data_url(url, in_flight, timeout) + except litellm.ImageFetchError as error: + return error + + +async def async_fetch_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> "Mapping[str, str | litellm.ImageFetchError]": + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + outcomes: Final = await asyncio.gather(*(_fetch_outcome(url, in_flight, timeout) for url in remote_urls)) + return MappingProxyType(dict(zip(remote_urls, outcomes, strict=True))) + + def _remote_urls_to_inline( messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] ) -> tuple[str, ...]: @@ -334,6 +351,10 @@ def _remote_urls_to_inline( return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote)))) +def remote_file_urls(messages: Iterable[AllMessageValues]) -> tuple[str, ...]: + return _remote_urls_to_inline(messages, inline_remote_file_urls) + + def inline_remote_media( messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues] should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url, diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 5b7c2a2c410..7ee017e9f22 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -33,6 +33,9 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None: class DeepSeekChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list: """ DeepSeek reasoner models support thinking parameter. diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py index 68b63812248..bd8e335b617 100644 --- a/litellm/llms/docker_model_runner/chat/transformation.py +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -25,6 +25,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig): The engine name (e.g., "llama.cpp") is part of the API endpoint path. """ + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index 002078dd04b..e9cfb5e09cf 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -22,6 +22,9 @@ class HerokuError(Exception): class HerokuChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index e152d56e470..f80a3bb8a5f 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -6,6 +6,7 @@ import asyncio import json import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from contextvars import ContextVar from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -33,9 +34,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, - async_inline_remote_media, + async_fetch_urls_to_base64, convert_url_to_base64, - inline_remote_file_urls, + remote_file_urls, ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -83,6 +84,19 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) +_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = frozenset({"user", "tool"}) +_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) +_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( + "prefetched_pdf_urls" +) + + +def _pdf_data_url(url: str) -> str: + prefetched: Final = _prefetched_pdf_urls.get(_NOTHING_PREFETCHED).get(url) + if isinstance(prefetched, litellm.ImageFetchError): + raise prefetched + return prefetched if prefetched is not None else convert_url_to_base64(url) + def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: if custom_llm_provider != "openai": @@ -302,7 +316,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): content_copy: Final = content_item.copy() file_id: Final = content_copy.get("file_id") if file_id is not None: - base64_data: Final = convert_url_to_base64(file_id) + base64_data: Final = _pdf_data_url(file_id) content_copy["file_data"] = base64_data content_copy["filename"] = "my_file.pdf" content_copy.pop("file_id") @@ -636,6 +650,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): **self._sanitized_tools_update_for_openai(optional_params, litellm_params), } + def _flattens_content_lists(self) -> bool: + return False + async def async_transform_request( self, model: str, @@ -644,13 +661,21 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: + pdf_urls: Final = remote_file_urls( + message for message in messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED + ) + if not pdf_urls: + return self.transform_request(model, messages, optional_params, litellm_params, headers) + prefetched: Final = ( + _NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls) + ) + reset_token: Final = _prefetched_pdf_urls.set(prefetched) try: - downloaded: Final = await async_inline_remote_media(messages, should_inline=inline_remote_file_urls) - except litellm.ImageFetchError: return await asyncio.to_thread( self.transform_request, model, messages, optional_params, litellm_params, headers ) - return self.transform_request(model, downloaded, optional_params, litellm_params, headers) + finally: + _prefetched_pdf_urls.reset(reset_token) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index d7aafe9f06e..24744a9f97c 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -28,6 +28,9 @@ def create_config_class(provider: SimpleProviderConfig): base_class: Final[type] = OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig class JSONProviderConfig(base_class): + def _flattens_content_lists(self) -> bool: + return bool(provider.special_handling.get("convert_content_list_to_string")) + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] @@ -47,7 +50,7 @@ def create_config_class(provider: SimpleProviderConfig): """Transform messages based on special_handling config""" # Handle content list to string conversion if configured - if provider.special_handling.get("convert_content_list_to_string"): + if self._flattens_content_lists(): messages = handle_messages_with_content_list_to_str_conversion(messages) if is_async: diff --git a/litellm/llms/sambanova/chat.py b/litellm/llms/sambanova/chat.py index 0350950f342..8247ec2770d 100644 --- a/litellm/llms/sambanova/chat.py +++ b/litellm/llms/sambanova/chat.py @@ -32,6 +32,9 @@ class SambanovaConfig(OpenAIGPTConfig): response_format: dict | None = None tools: list | None = None + def _flattens_content_lists(self) -> bool: + return True + def __init__( self, max_tokens: int | None = None, diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index f88f1829cb6..607a4aebb1e 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -6,11 +6,13 @@ import base64 import copy import itertools import json +import threading from collections.abc import Callable from typing import Final import pytest import respx +import httpx from httpx import Request, Response import litellm @@ -20,11 +22,16 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig +from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig +from litellm.llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig +from litellm.llms.heroku.chat.transformation import HerokuChatConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.llms.sambanova.chat import SambanovaConfig @@ -1415,16 +1422,15 @@ _ONCE_URL: Final = "https://files.example.test/download-once.pdf" ), ], ) -async def test_async_transform_request_downloads_a_pdf_url_once_and_leaves_the_callers_messages_unchanged( +async def test_async_transform_request_downloads_a_pdf_url_once_with_the_async_client( respx_mock: respx.MockRouter, messages: list[dict[str, object]], expected_messages: list[dict[str, object]] ) -> None: in_memory_cache.delete_cache(_ONCE_URL) download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) - sent: Final = copy.deepcopy(messages) request: Final = await OpenAIGPTConfig().async_transform_request( model="gpt-4.1", - messages=sent, + messages=copy.deepcopy(messages), optional_params={}, litellm_params={"custom_llm_provider": "openai"}, headers={}, @@ -1432,7 +1438,35 @@ async def test_async_transform_request_downloads_a_pdf_url_once_and_leaves_the_c assert download.call_count == 1 assert request["messages"] == expected_messages - assert sent == messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cache_once( + respx_mock: respx.MockRouter, +) -> None: + pdf_url: Final = "https://files.example.test/too-large-to-cache.pdf" + large_pdf: Final = _PDF_BYTES + b"0" * in_memory_cache.max_size_per_item * 1024 + + async def serve_large_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=large_pdf) + + download: Final = respx_mock.get(pdf_url).mock(side_effect=serve_large_pdf_to_async_clients_only) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"][0]["content"][1]["file"] == { + "file_data": f"data:application/pdf;base64,{base64.b64encode(large_pdf).decode('utf-8')}", + "filename": "my_file.pdf", + } @pytest.mark.usefixtures("pdf_urls_reachable") @@ -1509,28 +1543,23 @@ async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(r @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio -async def test_async_transform_request_rejects_a_file_part_without_its_file_and_leaves_the_callers_messages_unchanged( +async def test_async_transform_request_rejects_a_file_part_without_its_file_ahead_of_a_dead_pdf_url( respx_mock: respx.MockRouter, ) -> None: - pdf_url: Final = "https://files.example.test/before-a-broken-part.pdf" - in_memory_cache.delete_cache(pdf_url) - respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) - messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url), {"type": "file", "file": None}]}] - sent: Final = copy.deepcopy(messages) + pdf_url: Final = "https://files.example.test/after-a-broken-part.pdf" + respx_mock.get(pdf_url).mock(return_value=Response(404)) with pytest.raises( litellm.BadRequestError, match="Content block has type='file' but is missing the required 'file' field" ): await OpenAIGPTConfig().async_transform_request( model="gpt-4.1", - messages=sent, + messages=[{"role": "user", "content": [_SUMMARIZE, {"type": "file", "file": None}, _pdf_url_part(pdf_url)]}], optional_params={}, litellm_params={"custom_llm_provider": "openai"}, headers={}, ) - assert sent == messages - @pytest.mark.asyncio @pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"]) @@ -1548,24 +1577,214 @@ async def test_async_transform_request_sends_a_plain_string_tool_result_part_lik ) +def _json_provider_config(special_handling: dict[str, bool]) -> OpenAIGPTConfig: + provider: Final = SimpleProviderConfig( + "test-json-provider", + { + "base_url": "https://json-provider.example.test/v1", + "api_key_env": "TEST_JSON_PROVIDER_API_KEY", + "special_handling": special_handling, + }, + ) + return create_config_class(provider)() + + @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_still_flattens_for_a_provider_without_files_when_the_pdf_url_fails( - respx_mock: respx.MockRouter, +@pytest.mark.parametrize( + "config", + [ + pytest.param(SambanovaConfig(), id="sambanova"), + pytest.param(HerokuChatConfig(), id="heroku"), + pytest.param(DockerModelRunnerChatConfig(), id="docker_model_runner"), + pytest.param(_json_provider_config({"convert_content_list_to_string": True}), id="json_provider"), + pytest.param(DeepSeekChatConfig(), id="deepseek"), + ], +) +async def test_async_transform_request_does_not_download_a_pdf_url_for_a_provider_that_sends_only_text( + respx_mock: respx.MockRouter, config: OpenAIGPTConfig ) -> None: - pdf_url: Final = "https://files.example.test/missing.pdf" + pdf_url: Final = "https://files.example.test/dropped.pdf" in_memory_cache.delete_cache(pdf_url) - respx_mock.get(pdf_url).mock(return_value=Response(404)) - messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}] + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) - request: Final = await SambanovaConfig().async_transform_request( - model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + request: Final = await config.async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, ) + assert download.call_count == 0 assert request["messages"] == [{"role": "user", "content": "Summarize the report."}] +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_only_message_off_the_event_loop_for_a_provider_that_sends_only_text( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + event_loop_thread: Final = threading.get_ident() + + def serve_pdf_off_the_event_loop_only(request: Request) -> Response: + if threading.get_ident() == event_loop_thread: + return Response(503) + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + respx_mock.get(_ONCE_URL).mock(side_effect=serve_pdf_off_the_event_loop_only) + + request: Final = await HerokuChatConfig().async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_pdf_url_part(_ONCE_URL)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_url_with_the_async_client_for_a_json_provider_that_keeps_lists( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await _json_provider_config({}).async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + +def _serve_pdf(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _answer_not_found(request: Request) -> Response: + return Response(404) + + +def _refuse_connection(request: Request) -> Response: + raise httpx.ConnectError("connection refused", request=request) + + +_PARITY_URL: Final = "https://files.example.test/parity.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_PARITY_URL)]}], id="user_message"), + pytest.param( + [ + { + "role": "user", + "content": [_SUMMARIZE, {"type": "file", "file": {"file_id": _PARITY_URL, "filename": "report.pdf"}}], + } + ], + id="named_file", + ), + pytest.param(_tool_result_messages(_SUMMARIZE, _pdf_url_part(_PARITY_URL)), id="tool_result"), + pytest.param( + [ + {"role": "system", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": [_SUMMARIZE]}, + {"role": "assistant", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": "Go on."}, + ], + id="system_and_assistant_messages", + ), + ], +) +async def test_async_transform_request_matches_the_sync_request_downloads_and_callers_messages( + respx_mock: respx.MockRouter, messages: list[dict[str, object]] +) -> None: + download: Final = respx_mock.get(_PARITY_URL).mock(side_effect=_serve_pdf) + sync_messages: Final = copy.deepcopy(messages) + async_messages: Final = copy.deepcopy(messages) + + in_memory_cache.delete_cache(_PARITY_URL) + sync_request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=sync_messages, optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = download.call_count + in_memory_cache.delete_cache(_PARITY_URL) + async_request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=async_messages, optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request + assert async_messages == sync_messages + assert download.call_count - sync_downloads == sync_downloads + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dead_url_answer", [pytest.param(_answer_not_found, id="not_found"), pytest.param(_refuse_connection, id="refused")] +) +async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count( + respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response] +) -> None: + healthy_url: Final = "https://files.example.test/healthy.pdf" + dead_url: Final = "https://files.example.test/dead.pdf" + healthy: Final = respx_mock.get(healthy_url).mock(side_effect=_serve_pdf) + dead: Final = respx_mock.get(dead_url).mock(side_effect=dead_url_answer) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(healthy_url), _pdf_url_part(dead_url)]}] + + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = healthy.call_count + dead.call_count + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as async_error: + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert str(async_error.value) == str(sync_error.value) + assert healthy.call_count + dead.call_count - sync_downloads == sync_downloads + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_a_pdf_url_that_failed_in_an_async_request_downloads_again_in_a_later_request_of_the_same_task( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=[Response(404), _serve_pdf(Request("GET", _ONCE_URL))]) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}] + + with pytest.raises(litellm.ImageFetchError): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" From 0e143506acb2e53e468aa8ba8ba1c1b437d44360 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:07:09 -0700 Subject: [PATCH 4/8] fix(openai): skip up-front PDF downloads past a file part the transform rejects --- .../prompt_templates/image_handling.py | 21 ++++++++++---- .../chat/test_openai_gpt_transformation.py | 29 ++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 41a5f586429..6930063f127 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -7,7 +7,7 @@ import base64 import time from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass -from itertools import chain +from itertools import chain, takewhile from types import MappingProxyType from typing import Final @@ -343,16 +343,25 @@ async def async_fetch_urls_to_base64( return MappingProxyType(dict(zip(remote_urls, outcomes, strict=True))) -def _remote_urls_to_inline( - messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] -) -> tuple[str, ...]: - parts: Final = chain.from_iterable(_content_parts(message) for message in messages) +def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: remotes: Final = (remote for part in parts if (remote := _parse_remote_part(part)) is not None) return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote)))) +def _remote_urls_to_inline( + messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] +) -> tuple[str, ...]: + return _remote_urls_in_parts(chain.from_iterable(_content_parts(message) for message in messages), should_inline) + + +def _is_not_a_file_part_missing_its_file(part: object) -> bool: + fields: Final = _as_mapping(part) + return fields is None or fields.get("type") != "file" or fields.get("file") is not None + + def remote_file_urls(messages: Iterable[AllMessageValues]) -> tuple[str, ...]: - return _remote_urls_to_inline(messages, inline_remote_file_urls) + parts: Final = chain.from_iterable(_content_parts(message) for message in messages) + return _remote_urls_in_parts(takewhile(_is_not_a_file_part_missing_its_file, parts), inline_remote_file_urls) def inline_remote_media( diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index ef520f8e9b8..375a125d51e 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_MEDIA_BOUNDARY, ) from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache +from litellm.llms.bedrock.chat.chat_completions.transformation import AmazonBedrockRuntimeChatCompletionsConfig from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig from litellm.llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig @@ -33,6 +34,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( from litellm.llms.openai_like.dynamic_config import create_config_class from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.llms.sambanova.chat import SambanovaConfig +from litellm.llms.vertex_ai.vertex_gemma_models.transformation import VertexGemmaConfig class TestOpenAIGPTConfig: @@ -1541,13 +1543,14 @@ async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(r @pytest.mark.usefixtures("pdf_urls_reachable") -@pytest.mark.respx(assert_all_called=True) +@pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_rejects_a_file_part_without_its_file_ahead_of_a_dead_pdf_url( +async def test_async_transform_request_rejects_a_file_part_without_its_file_before_downloading_a_later_pdf_url( respx_mock: respx.MockRouter, ) -> None: pdf_url: Final = "https://files.example.test/after-a-broken-part.pdf" - respx_mock.get(pdf_url).mock(return_value=Response(404)) + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(return_value=Response(200, content=b"%PDF-1.4")) with pytest.raises( litellm.BadRequestError, match="Content block has type='file' but is missing the required 'file' field" @@ -1560,6 +1563,8 @@ async def test_async_transform_request_rejects_a_file_part_without_its_file_ahea headers={}, ) + assert not download.called + @pytest.mark.asyncio @pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"]) @@ -1711,21 +1716,29 @@ _PARITY_URL: Final = "https://files.example.test/parity.pdf" ), ], ) +@pytest.mark.parametrize( + ("config", "model"), + [ + pytest.param(OpenAIGPTConfig(), "gpt-4.1", id="openai"), + pytest.param(AmazonBedrockRuntimeChatCompletionsConfig(), "us.xai.grok-4.6", id="bedrock_chat_completions"), + pytest.param(VertexGemmaConfig(), "gemma-3-27b-it", id="vertex_gemma"), + ], +) async def test_async_transform_request_matches_the_sync_request_downloads_and_callers_messages( - respx_mock: respx.MockRouter, messages: list[dict[str, object]] + respx_mock: respx.MockRouter, messages: list[dict[str, object]], config: OpenAIGPTConfig, model: str ) -> None: download: Final = respx_mock.get(_PARITY_URL).mock(side_effect=_serve_pdf) sync_messages: Final = copy.deepcopy(messages) async_messages: Final = copy.deepcopy(messages) in_memory_cache.delete_cache(_PARITY_URL) - sync_request: Final = OpenAIGPTConfig().transform_request( - model="gpt-4.1", messages=sync_messages, optional_params={}, litellm_params={}, headers={} + sync_request: Final = config.transform_request( + model=model, messages=sync_messages, optional_params={}, litellm_params={}, headers={} ) sync_downloads: Final = download.call_count in_memory_cache.delete_cache(_PARITY_URL) - async_request: Final = await OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", messages=async_messages, optional_params={}, litellm_params={}, headers={} + async_request: Final = await config.async_transform_request( + model=model, messages=async_messages, optional_params={}, litellm_params={}, headers={} ) assert async_request == sync_request From b48f2bf1c76705f698b57817ee020ca64718582d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:32:55 -0700 Subject: [PATCH 5/8] fix(openai): keep system-message PDF downloads off the event loop and stop downloads after a dead PDF URL --- .../prompt_templates/image_handling.py | 25 ++++++- .../llms/openai/chat/gpt_transformation.py | 5 +- .../chat/test_openai_gpt_transformation.py | 73 +++++++++++++++++-- 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 6930063f127..a3eb9cb58e0 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,7 +5,7 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import AsyncIterator, Callable, Iterable, Mapping from dataclasses import dataclass from itertools import chain, takewhile from types import MappingProxyType @@ -335,12 +335,27 @@ async def _fetch_outcome( return error +async def _outcomes_through_first_failure( + downloads: "tuple[asyncio.Task[str | litellm.ImageFetchError], ...]", +) -> "AsyncIterator[str | litellm.ImageFetchError]": + for download in downloads: + yield await download + if isinstance(download.result(), litellm.ImageFetchError): + return + + async def async_fetch_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> "Mapping[str, str | litellm.ImageFetchError]": in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - outcomes: Final = await asyncio.gather(*(_fetch_outcome(url, in_flight, timeout) for url in remote_urls)) - return MappingProxyType(dict(zip(remote_urls, outcomes, strict=True))) + downloads: Final = tuple(asyncio.create_task(_fetch_outcome(url, in_flight, timeout)) for url in remote_urls) + try: + outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) + finally: + for download in downloads: + download.cancel() + await asyncio.gather(*downloads, return_exceptions=True) + return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: @@ -359,6 +374,10 @@ def _is_not_a_file_part_missing_its_file(part: object) -> bool: return fields is None or fields.get("type") != "file" or fields.get("file") is not None +def has_remote_file_urls(messages: Iterable[AllMessageValues]) -> bool: + return bool(_remote_urls_to_inline(messages, inline_remote_file_urls)) + + def remote_file_urls(messages: Iterable[AllMessageValues]) -> tuple[str, ...]: parts: Final = chain.from_iterable(_content_parts(message) for message in messages) return _remote_urls_in_parts(takewhile(_is_not_a_file_part_missing_its_file, parts), inline_remote_file_urls) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index df5699f165f..7aeebfc1f8b 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, async_fetch_urls_to_base64, convert_url_to_base64, + has_remote_file_urls, remote_file_urls, ) from litellm.litellm_core_utils.url_utils import join_api_base @@ -694,11 +695,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: + if not has_remote_file_urls(messages): + return self.transform_request(model, messages, optional_params, litellm_params, headers) pdf_urls: Final = remote_file_urls( message for message in messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED ) - if not pdf_urls: - return self.transform_request(model, messages, optional_params, litellm_params, headers) prefetched: Final = ( _NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls) ) diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 375a125d51e..38ef6a0f4c5 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -31,6 +31,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig from litellm.llms.openai_like.dynamic_config import create_config_class from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.llms.sambanova.chat import SambanovaConfig @@ -1629,8 +1630,28 @@ async def test_async_transform_request_does_not_download_a_pdf_url_for_a_provide @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio -async def test_async_transform_request_downloads_a_pdf_only_message_off_the_event_loop_for_a_provider_that_sends_only_text( +@pytest.mark.parametrize( + ("config", "messages", "expected_messages"), + [ + pytest.param( + HerokuChatConfig(), + [{"role": "user", "content": [_pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}], + id="pdf_only_message_for_a_provider_that_sends_only_text", + ), + pytest.param( + OpenAIOSeriesConfig(), + [{"role": "system", "content": [_pdf_url_part(_ONCE_URL)]}, {"role": "user", "content": [_SUMMARIZE]}], + [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}, {"role": "user", "content": [_SUMMARIZE]}], + id="system_message_for_a_model_without_system_messages", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_the_sync_transform_reaches_off_the_event_loop( respx_mock: respx.MockRouter, + config: OpenAIGPTConfig, + messages: list[dict[str, object]], + expected_messages: list[dict[str, object]], ) -> None: in_memory_cache.delete_cache(_ONCE_URL) event_loop_thread: Final = threading.get_ident() @@ -1642,15 +1663,15 @@ async def test_async_transform_request_downloads_a_pdf_only_message_off_the_even respx_mock.get(_ONCE_URL).mock(side_effect=serve_pdf_off_the_event_loop_only) - request: Final = await HerokuChatConfig().async_transform_request( - model="test-model", - messages=[{"role": "user", "content": [_pdf_url_part(_ONCE_URL)]}], + request: Final = await config.async_transform_request( + model="test-model-without-system-messages", + messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}, ) - assert request["messages"] == [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}] + assert request["messages"] == expected_messages @pytest.mark.usefixtures("pdf_urls_reachable") @@ -1777,6 +1798,48 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ assert healthy.call_count + dead.call_count - sync_downloads == sync_downloads +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_the_downloads_after_it( + respx_mock: respx.MockRouter, +) -> None: + dead_url: Final = "https://files.example.test/dead-first.pdf" + slow_url: Final = "https://files.example.test/slow-after-dead.pdf" + in_memory_cache.delete_cache(slow_url) + slow_download_started: Final = asyncio.Event() + slow_download_stopped: Final = asyncio.Event() + + async def answer_not_found_once_the_slow_download_started(request: Request) -> Response: + await asyncio.wait_for(slow_download_started.wait(), timeout=5) + return Response(404) + + async def serve_pdf_never(request: Request) -> Response: + slow_download_started.set() + try: + await asyncio.Event().wait() + finally: + slow_download_stopped.set() + return _serve_pdf(request) + + respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started) + respx_mock.get(slow_url).mock(side_effect=serve_pdf_never) + + with pytest.raises(litellm.ImageFetchError): + await asyncio.wait_for( + OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(slow_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ), + timeout=5, + ) + + assert slow_download_stopped.is_set() + + @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio From 2c5ca7a78eb2a6d2c28ebdecd2c4621aed82ac95 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:34:26 -0700 Subject: [PATCH 6/8] fix(openai): let PDF downloads after a dead URL finish and retry rate-limited ones one at a time Cancelling in-flight downloads after a dead PDF URL could abort a proxy worker on uvloop, so the downloads now run to the end in the background. A URL whose up-front download got a 429 downloads again inside the transform one at a time, as on main. A content part whose type is not a string is no longer read as a PDF part, so it reaches the provider instead of raising a TypeError. --- litellm/exceptions.py | 2 + .../prompt_templates/image_handling.py | 46 +++++++----- .../llms/openai/chat/gpt_transformation.py | 2 +- .../chat/test_openai_gpt_transformation.py | 70 ++++++++++++++++--- 4 files changed, 92 insertions(+), 28 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 12425c45b01..efd8a1323fe 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -328,7 +328,9 @@ class ImageFetchError(BadRequestError): max_retries: int | None = None, num_retries: int | None = None, body: Mapping[str, object] | None = None, + url_status_code: int | None = None, ): + self.url_status_code = url_status_code super().__init__( message=message, model=model, diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index a3eb9cb58e0..6385db9a742 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -7,9 +7,10 @@ import base64 import time from collections.abc import AsyncIterator, Callable, Iterable, Mapping from dataclasses import dataclass +from http import HTTPStatus from itertools import chain, takewhile from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx from httpx import Response @@ -39,7 +40,8 @@ def _header_value(response: Response, name: str) -> str | None: def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: raise litellm.ImageFetchError( - f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}" + f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}", + url_status_code=response.status_code, ) # Check size before downloading if Content-Length header is present @@ -229,7 +231,9 @@ def _parse_remote_file(fields: Mapping[str, object]) -> _RemoteFile | None: def _parse_remote_source(fields: Mapping[str, object]) -> _RemoteSource | None: - source: Final = _as_mapping(fields.get("source")) if fields.get("type") in _ANTHROPIC_MEDIA_BLOCK_TYPES else None + part_type: Final = fields.get("type") + is_media_block: Final = isinstance(part_type, str) and part_type in _ANTHROPIC_MEDIA_BLOCK_TYPES + source: Final = _as_mapping(fields.get("source")) if is_media_block else None url: Final = _remote_url(source.get("url")) if source is not None and source.get("type") == "url" else None return _RemoteSource(fields, source, url) if source is not None and url is not None else None @@ -326,21 +330,31 @@ async def async_convert_urls_to_base64( raise +_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None" +_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop + + async def _fetch_outcome( url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None -) -> "str | litellm.ImageFetchError": +) -> _FetchOutcome: try: return await _fetch_data_url(url, in_flight, timeout) except litellm.ImageFetchError as error: - return error + return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error + + +def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]": + _UNREAD_DOWNLOADS.add(download) + download.add_done_callback(_UNREAD_DOWNLOADS.discard) + return download async def _outcomes_through_first_failure( - downloads: "tuple[asyncio.Task[str | litellm.ImageFetchError], ...]", -) -> "AsyncIterator[str | litellm.ImageFetchError]": + downloads: "tuple[asyncio.Task[_FetchOutcome], ...]", +) -> AsyncIterator[_FetchOutcome]: for download in downloads: - yield await download - if isinstance(download.result(), litellm.ImageFetchError): + yield (outcome := await asyncio.shield(download)) + if isinstance(outcome, litellm.ImageFetchError): return @@ -348,14 +362,12 @@ async def async_fetch_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> "Mapping[str, str | litellm.ImageFetchError]": in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - downloads: Final = tuple(asyncio.create_task(_fetch_outcome(url, in_flight, timeout)) for url in remote_urls) - try: - outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) - finally: - for download in downloads: - download.cancel() - await asyncio.gather(*downloads, return_exceptions=True) - return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) + downloads: Final = tuple( + _kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls + ) + outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) + downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True) + return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None}) def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 7aeebfc1f8b..4efa86ad637 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -85,7 +85,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) -_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = frozenset({"user", "tool"}) +_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool") _NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) _prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( "prefetched_pdf_urls" diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 38ef6a0f4c5..4f2a99177ac 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -10,9 +10,9 @@ import threading from collections.abc import Callable from typing import Final +import httpx import pytest import respx -import httpx from httpx import Request, Response import litellm @@ -1801,29 +1801,28 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_the_downloads_after_it( +async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it( respx_mock: respx.MockRouter, ) -> None: dead_url: Final = "https://files.example.test/dead-first.pdf" slow_url: Final = "https://files.example.test/slow-after-dead.pdf" in_memory_cache.delete_cache(slow_url) slow_download_started: Final = asyncio.Event() - slow_download_stopped: Final = asyncio.Event() + slow_download_released: Final = asyncio.Event() + slow_download_served: Final = asyncio.Event() async def answer_not_found_once_the_slow_download_started(request: Request) -> Response: await asyncio.wait_for(slow_download_started.wait(), timeout=5) return Response(404) - async def serve_pdf_never(request: Request) -> Response: + async def serve_pdf_once_released(request: Request) -> Response: slow_download_started.set() - try: - await asyncio.Event().wait() - finally: - slow_download_stopped.set() + await slow_download_released.wait() + slow_download_served.set() return _serve_pdf(request) respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started) - respx_mock.get(slow_url).mock(side_effect=serve_pdf_never) + respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released) with pytest.raises(litellm.ImageFetchError): await asyncio.wait_for( @@ -1836,8 +1835,59 @@ async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_ ), timeout=5, ) + slow_download_released.set() + await asyncio.wait_for(slow_download_served.wait(), timeout=5) + await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) - assert slow_download_stopped.is_set() + assert in_memory_cache.get_cache(slow_url) is not None + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time( + respx_mock: respx.MockRouter, +) -> None: + pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2)) + downloads: Final = tuple( + respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls + ) + for url in pdf_urls: + in_memory_cache.delete_cache(url) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [ + {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]} + ] + assert [download.call_count for download in downloads] == [2, 2] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", ["system", "assistant", "developer"]) +@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")]) +async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request( + role: str, part_type: object +) -> None: + messages: Final = [ + {"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]}, + {"role": "user", "content": "hi"}, + ] + + sync_request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + async_request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request @pytest.mark.usefixtures("pdf_urls_reachable") From 8b9f55a4b04054f99decac60105eac5d04c83e54 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:05:15 -0700 Subject: [PATCH 7/8] fix(openai): download a request's PDF URLs one at a time and stop at the first dead one --- litellm/exceptions.py | 2 - .../prompt_templates/image_handling.py | 43 ++----- .../chat/test_openai_gpt_transformation.py | 108 ++++++------------ 3 files changed, 47 insertions(+), 106 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index efd8a1323fe..12425c45b01 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -328,9 +328,7 @@ class ImageFetchError(BadRequestError): max_retries: int | None = None, num_retries: int | None = None, body: Mapping[str, object] | None = None, - url_status_code: int | None = None, ): - self.url_status_code = url_status_code super().__init__( message=message, model=model, diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 6385db9a742..e49afbb07b9 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -7,10 +7,9 @@ import base64 import time from collections.abc import AsyncIterator, Callable, Iterable, Mapping from dataclasses import dataclass -from http import HTTPStatus from itertools import chain, takewhile from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final import httpx from httpx import Response @@ -40,8 +39,7 @@ def _header_value(response: Response, name: str) -> str | None: def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: raise litellm.ImageFetchError( - f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}", - url_status_code=response.status_code, + f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}" ) # Check size before downloading if Content-Length header is present @@ -330,30 +328,18 @@ async def async_convert_urls_to_base64( raise -_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None" -_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop - - -async def _fetch_outcome( - url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None -) -> _FetchOutcome: +async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError": try: - return await _fetch_data_url(url, in_flight, timeout) + return await async_convert_url_to_base64(url, timeout=timeout) except litellm.ImageFetchError as error: - return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error + return error -def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]": - _UNREAD_DOWNLOADS.add(download) - download.add_done_callback(_UNREAD_DOWNLOADS.discard) - return download - - -async def _outcomes_through_first_failure( - downloads: "tuple[asyncio.Task[_FetchOutcome], ...]", -) -> AsyncIterator[_FetchOutcome]: - for download in downloads: - yield (outcome := await asyncio.shield(download)) +async def _downloads_through_first_failure( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None +) -> "AsyncIterator[str | litellm.ImageFetchError]": + for url in remote_urls: + yield (outcome := await _download_outcome(url, timeout)) if isinstance(outcome, litellm.ImageFetchError): return @@ -361,13 +347,8 @@ async def _outcomes_through_first_failure( async def async_fetch_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> "Mapping[str, str | litellm.ImageFetchError]": - in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - downloads: Final = tuple( - _kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls - ) - outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) - downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True) - return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None}) + outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)]) + return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 4f2a99177ac..46c99df463b 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -4,7 +4,6 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation import asyncio import base64 import copy -import itertools import json import threading from collections.abc import Callable @@ -1475,25 +1474,24 @@ async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cac @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio -async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_the_same_time( +async def test_async_transform_request_downloads_pdf_urls_one_at_a_time_for_a_host_that_refuses_bursts( respx_mock: respx.MockRouter, ) -> None: - pdf_urls: Final = tuple(f"https://files.example.test/together-{index}.pdf" for index in range(3)) + pdf_urls: Final = tuple(f"https://files.example.test/in-turn-{index}.pdf" for index in range(3)) for pdf_url in pdf_urls: in_memory_cache.delete_cache(pdf_url) - started_downloads: Final = itertools.count(1) - every_download_started: Final = asyncio.Event() + download_running: Final = asyncio.Lock() - async def serve_pdf_once_every_download_started(request: Request) -> Response: - if next(started_downloads) == len(pdf_urls): - every_download_started.set() - try: - await asyncio.wait_for(every_download_started.wait(), timeout=5) - except TimeoutError as error: - raise litellm.ImageFetchError("the PDF URLs were downloaded one after another") from error - return await _serve_pdf_to_async_clients_only(request) + async def refuse_a_second_download_at_once(request: Request) -> Response: + if download_running.locked(): + return Response(429) + async with download_running: + await asyncio.sleep(0.05) + return await _serve_pdf_to_async_clients_only(request) - respx_mock.route(method="GET", host="files.example.test").mock(side_effect=serve_pdf_once_every_download_started) + downloads: Final = respx_mock.route(method="GET", host="files.example.test").mock( + side_effect=refuse_a_second_download_at_once + ) request: Final = await OpenAIGPTConfig().async_transform_request( model="gpt-4.1", @@ -1504,6 +1502,7 @@ async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_ ) assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}] + assert downloads.call_count == len(pdf_urls) @pytest.mark.usefixtures("pdf_urls_reachable") @@ -1702,6 +1701,10 @@ def _answer_not_found(request: Request) -> Response: return Response(404) +def _answer_too_many_requests(request: Request) -> Response: + return Response(429) + + def _refuse_connection(request: Request) -> Response: raise httpx.ConnectError("connection refused", request=request) @@ -1771,7 +1774,12 @@ async def test_async_transform_request_matches_the_sync_request_downloads_and_ca @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio @pytest.mark.parametrize( - "dead_url_answer", [pytest.param(_answer_not_found, id="not_found"), pytest.param(_refuse_connection, id="refused")] + "dead_url_answer", + [ + pytest.param(_answer_not_found, id="not_found"), + pytest.param(_answer_too_many_requests, id="rate_limited"), + pytest.param(_refuse_connection, id="refused"), + ], ) async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count( respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response] @@ -1801,72 +1809,26 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it( +async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it( respx_mock: respx.MockRouter, ) -> None: dead_url: Final = "https://files.example.test/dead-first.pdf" - slow_url: Final = "https://files.example.test/slow-after-dead.pdf" - in_memory_cache.delete_cache(slow_url) - slow_download_started: Final = asyncio.Event() - slow_download_released: Final = asyncio.Event() - slow_download_served: Final = asyncio.Event() - - async def answer_not_found_once_the_slow_download_started(request: Request) -> Response: - await asyncio.wait_for(slow_download_started.wait(), timeout=5) - return Response(404) - - async def serve_pdf_once_released(request: Request) -> Response: - slow_download_started.set() - await slow_download_released.wait() - slow_download_served.set() - return _serve_pdf(request) - - respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started) - respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released) + later_url: Final = "https://files.example.test/after-dead.pdf" + in_memory_cache.delete_cache(later_url) + respx_mock.get(dead_url).mock(return_value=Response(404)) + later: Final = respx_mock.get(later_url).mock(side_effect=_serve_pdf_to_async_clients_only) with pytest.raises(litellm.ImageFetchError): - await asyncio.wait_for( - OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(slow_url)]}], - optional_params={}, - litellm_params={}, - headers={}, - ), - timeout=5, + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(later_url)]}], + optional_params={}, + litellm_params={}, + headers={}, ) - slow_download_released.set() - await asyncio.wait_for(slow_download_served.wait(), timeout=5) await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) - assert in_memory_cache.get_cache(slow_url) is not None - - -@pytest.mark.usefixtures("pdf_urls_reachable") -@pytest.mark.respx(assert_all_called=True) -@pytest.mark.asyncio -async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time( - respx_mock: respx.MockRouter, -) -> None: - pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2)) - downloads: Final = tuple( - respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls - ) - for url in pdf_urls: - in_memory_cache.delete_cache(url) - - request: Final = await OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}], - optional_params={}, - litellm_params={}, - headers={}, - ) - - assert request["messages"] == [ - {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]} - ] - assert [download.call_count for download in downloads] == [2, 2] + assert later.call_count == 0 @pytest.mark.asyncio From c9d1c20b635ed76b26ad9d80a55bd288dcab6c1f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:24:09 -0700 Subject: [PATCH 8/8] fix(openai): prefetch the PDF URLs the sync transform downloads, after role mapping and up to the first unreadable part --- .../prompt_templates/image_handling.py | 52 ++++--- .../llms/openai/chat/gpt_transformation.py | 21 ++- .../openai/chat/o_series_transformation.py | 17 ++- .../chat/test_openai_gpt_transformation.py | 137 +++++++++++++++--- 4 files changed, 169 insertions(+), 58 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index e49afbb07b9..1c3c5691d06 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,7 +5,7 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import AsyncIterator, Callable, Iterable, Mapping +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping from dataclasses import dataclass from itertools import chain, takewhile from types import MappingProxyType @@ -209,10 +209,6 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool: return media.part_type == "image_url" -def inline_remote_file_urls(media: RemoteMedia) -> bool: - return media.part_type == "file" - - def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None: if fields.get("type") != "image_url": return None @@ -229,9 +225,7 @@ def _parse_remote_file(fields: Mapping[str, object]) -> _RemoteFile | None: def _parse_remote_source(fields: Mapping[str, object]) -> _RemoteSource | None: - part_type: Final = fields.get("type") - is_media_block: Final = isinstance(part_type, str) and part_type in _ANTHROPIC_MEDIA_BLOCK_TYPES - source: Final = _as_mapping(fields.get("source")) if is_media_block else None + source: Final = _as_mapping(fields.get("source")) if fields.get("type") in _ANTHROPIC_MEDIA_BLOCK_TYPES else None url: Final = _remote_url(source.get("url")) if source is not None and source.get("type") == "url" else None return _RemoteSource(fields, source, url) if source is not None and url is not None else None @@ -351,29 +345,47 @@ async def async_fetch_urls_to_base64( return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) -def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: +def _remote_urls_to_inline( + messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] +) -> tuple[str, ...]: + parts: Final = chain.from_iterable(_content_parts(message) for message in messages) remotes: Final = (remote for part in parts if (remote := _parse_remote_part(part)) is not None) return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote)))) -def _remote_urls_to_inline( - messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] -) -> tuple[str, ...]: - return _remote_urls_in_parts(chain.from_iterable(_content_parts(message) for message in messages), should_inline) +def _message_parts(messages: Iterable[AllMessageValues]) -> Iterator[object]: + return chain.from_iterable(_content_parts(message) for message in messages) -def _is_not_a_file_part_missing_its_file(part: object) -> bool: +def _file_of_file_part(part: object) -> Mapping[str, object] | None: fields: Final = _as_mapping(part) - return fields is None or fields.get("type") != "file" or fields.get("file") is not None + return _as_mapping(fields.get("file")) if fields is not None and fields.get("type") == "file" else None -def has_remote_file_urls(messages: Iterable[AllMessageValues]) -> bool: - return bool(_remote_urls_to_inline(messages, inline_remote_file_urls)) +def _file_id_url(part: object, url_starts: tuple[str, ...]) -> str | None: + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file_id if isinstance(file_id, str) and file_id.startswith(url_starts) else None -def remote_file_urls(messages: Iterable[AllMessageValues]) -> tuple[str, ...]: - parts: Final = chain.from_iterable(_content_parts(message) for message in messages) - return _remote_urls_in_parts(takewhile(_is_not_a_file_part_missing_its_file, parts), inline_remote_file_urls) +def _is_readable_file_part_or_other_part(part: object) -> bool: + fields: Final = _as_mapping(part) + if fields is None or fields.get("type") != "file": + return fields is not None + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file is not None and (not file_id or isinstance(file_id, str)) + + +def has_file_id_urls(messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]) -> bool: + return any(_file_id_url(part, url_starts) is not None for part in _message_parts(messages)) + + +def file_id_urls_before_an_unreadable_part( + messages: Iterable[AllMessageValues], url_starts: tuple[str, ...] +) -> tuple[str, ...]: + readable_parts: Final = takewhile(_is_readable_file_part_or_other_part, _message_parts(messages)) + return tuple(dict.fromkeys(url for part in readable_parts if (url := _file_id_url(part, url_starts)) is not None)) def inline_remote_media( diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4efa86ad637..5c6b3aed40e 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,8 +36,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, async_fetch_urls_to_base64, convert_url_to_base64, - has_remote_file_urls, - remote_file_urls, + file_id_urls_before_an_unreadable_part, + has_file_id_urls, ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -86,6 +86,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) _ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool") +_PDF_URL_STARTS: Final = ("https://", "http://", "www.") _NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) _prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( "prefetched_pdf_urls" @@ -330,9 +331,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool: - potential_pdf_url_starts: Final = ["https://", "http://", "www."] file_id: Final = content_item.get("file_id") - if file_id and any(file_id.startswith(start) for start in potential_pdf_url_starts): + if file_id and any(file_id.startswith(start) for start in _PDF_URL_STARTS): return True return False @@ -687,6 +687,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): def _flattens_content_lists(self) -> bool: return False + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + return messages + async def async_transform_request( self, model: str, @@ -695,10 +698,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - if not has_remote_file_urls(messages): + if not has_file_id_urls(messages, _PDF_URL_STARTS): return self.transform_request(model, messages, optional_params, litellm_params, headers) - pdf_urls: Final = remote_file_urls( - message for message in messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED + sent_messages: Final = self._with_provider_roles( + self._prompt_cache_ordered_messages(messages, litellm_params), model + ) + pdf_urls: Final = file_id_urls_before_an_unreadable_part( + (message for message in sent_messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED), + _PDF_URL_STARTS, ) prefetched: Final = ( _NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls) diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 4d56a3b38c5..06257af7075 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -122,6 +122,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): return super()._map_openai_params(non_default_params, optional_params, model, drop_params) + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + keeps_system_messages: Final = supports_system_messages(model, "openai") + return tuple( + message + if keeps_system_messages or message["role"] != "system" + else ChatCompletionUserMessage(content=message["content"], role="user") + for message in messages + ) + def is_model_o_series_model(self, model: str) -> bool: model = model.split("/")[-1] # could be "openai/o3" or "o3" return ( @@ -152,13 +161,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - modalities: image => drop param (if user opts in to dropping param) - role: system ==> translate to role 'user' """ - _supports_system_messages: Final = supports_system_messages(model, "openai") - transformed_messages: Final = list(messages) - for i, message in enumerate(transformed_messages): - if message["role"] == "system" and not _supports_system_messages: - new_message = ChatCompletionUserMessage(content=message["content"], role="user") - transformed_messages[i] = new_message # Replace the old message with the new one - + transformed_messages: Final = list(self._with_provider_roles(messages, model)) if is_async: return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 46c99df463b..e70c2b60305 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1545,19 +1545,39 @@ async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(r @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_rejects_a_file_part_without_its_file_before_downloading_a_later_pdf_url( - respx_mock: respx.MockRouter, +@pytest.mark.parametrize( + ("unreadable_part", "error"), + [ + pytest.param( + {"type": "file", "file": None}, + "Content block has type='file' but is missing the required 'file' field", + id="file_part_without_its_file", + ), + pytest.param( + {"type": "file", "file": "file-uploaded"}, "has no attribute 'items'", id="file_that_is_not_an_object" + ), + pytest.param( + {"type": "file", "file": {"file_id": 123}}, "has no attribute 'startswith'", id="file_id_not_a_string" + ), + pytest.param("Summarize the report.", "has no attribute 'get'", id="part_that_is_not_an_object"), + ], +) +async def test_async_transform_request_rejects_an_unreadable_part_like_the_sync_transform_before_a_later_pdf_url( + respx_mock: respx.MockRouter, unreadable_part: object, error: str ) -> None: pdf_url: Final = "https://files.example.test/after-a-broken-part.pdf" in_memory_cache.delete_cache(pdf_url) download: Final = respx_mock.get(pdf_url).mock(return_value=Response(200, content=b"%PDF-1.4")) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, unreadable_part, _pdf_url_part(pdf_url)]}] - with pytest.raises( - litellm.BadRequestError, match="Content block has type='file' but is missing the required 'file' field" - ): + with pytest.raises((litellm.BadRequestError, AttributeError), match=error) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + with pytest.raises(type(sync_error.value), match=error): await OpenAIGPTConfig().async_transform_request( model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, {"type": "file", "file": None}, _pdf_url_part(pdf_url)]}], + messages=copy.deepcopy(messages), optional_params={}, litellm_params={"custom_llm_provider": "openai"}, headers={}, @@ -1638,12 +1658,6 @@ async def test_async_transform_request_does_not_download_a_pdf_url_for_a_provide [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}], id="pdf_only_message_for_a_provider_that_sends_only_text", ), - pytest.param( - OpenAIOSeriesConfig(), - [{"role": "system", "content": [_pdf_url_part(_ONCE_URL)]}, {"role": "user", "content": [_SUMMARIZE]}], - [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}, {"role": "user", "content": [_SUMMARIZE]}], - id="system_message_for_a_model_without_system_messages", - ), ], ) async def test_async_transform_request_downloads_a_pdf_the_sync_transform_reaches_off_the_event_loop( @@ -1673,6 +1687,30 @@ async def test_async_transform_request_downloads_a_pdf_the_sync_transform_reache assert request["messages"] == expected_messages +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_system_pdf_with_the_async_client_for_a_model_without_system_messages( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[{"role": "system", "content": [_pdf_url_part(_ONCE_URL)]}, {"role": "user", "content": [_SUMMARIZE]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == [ + {"role": "user", "content": [_DOWNLOADED_PDF_PART]}, + {"role": "user", "content": [_SUMMARIZE]}, + ] + + @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio @@ -1806,31 +1844,82 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ assert healthy.call_count + dead.call_count - sync_downloads == sync_downloads +_DEAD_URL: Final = "https://files.example.test/dead-first.pdf" +_AFTER_DEAD_URL: Final = "https://files.example.test/after-dead.pdf" +_WWW_FILE_ID: Final = "www.files.example.test/no-scheme.pdf" + + @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "model", "messages"), + [ + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_DEAD_URL), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="dead_url", + ), + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_WWW_FILE_ID), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="www_file_id", + ), + pytest.param( + OpenAIOSeriesConfig(), + "test-model-without-system-messages", + [ + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + ], + id="system_message_for_a_model_without_system_messages", + ), + ], +) async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it( - respx_mock: respx.MockRouter, + respx_mock: respx.MockRouter, config: OpenAIGPTConfig, model: str, messages: list[dict[str, object]] ) -> None: - dead_url: Final = "https://files.example.test/dead-first.pdf" - later_url: Final = "https://files.example.test/after-dead.pdf" - in_memory_cache.delete_cache(later_url) - respx_mock.get(dead_url).mock(return_value=Response(404)) - later: Final = respx_mock.get(later_url).mock(side_effect=_serve_pdf_to_async_clients_only) + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) with pytest.raises(litellm.ImageFetchError): - await OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(later_url)]}], - optional_params={}, - litellm_params={}, - headers={}, + await config.async_transform_request( + model=model, messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} ) await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) assert later.call_count == 0 +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_moved_system_pdf_first_when_system_messages_go_first( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + with pytest.raises(litellm.ImageFetchError): + await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[ + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + ], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert later.call_count == 0 + + @pytest.mark.asyncio @pytest.mark.parametrize("role", ["system", "assistant", "developer"]) @pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")])