From 2c5ca7a78eb2a6d2c28ebdecd2c4621aed82ac95 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:34:26 -0700 Subject: [PATCH] fix(openai): let PDF downloads after a dead URL finish and retry rate-limited ones one at a time Cancelling in-flight downloads after a dead PDF URL could abort a proxy worker on uvloop, so the downloads now run to the end in the background. A URL whose up-front download got a 429 downloads again inside the transform one at a time, as on main. A content part whose type is not a string is no longer read as a PDF part, so it reaches the provider instead of raising a TypeError. --- litellm/exceptions.py | 2 + .../prompt_templates/image_handling.py | 46 +++++++----- .../llms/openai/chat/gpt_transformation.py | 2 +- .../chat/test_openai_gpt_transformation.py | 70 ++++++++++++++++--- 4 files changed, 92 insertions(+), 28 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 12425c45b01..efd8a1323fe 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -328,7 +328,9 @@ class ImageFetchError(BadRequestError): max_retries: int | None = None, num_retries: int | None = None, body: Mapping[str, object] | None = None, + url_status_code: int | None = None, ): + self.url_status_code = url_status_code super().__init__( message=message, model=model, diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index a3eb9cb58e0..6385db9a742 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -7,9 +7,10 @@ import base64 import time from collections.abc import AsyncIterator, Callable, Iterable, Mapping from dataclasses import dataclass +from http import HTTPStatus from itertools import chain, takewhile from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx from httpx import Response @@ -39,7 +40,8 @@ def _header_value(response: Response, name: str) -> str | None: def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: raise litellm.ImageFetchError( - f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}" + f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}", + url_status_code=response.status_code, ) # Check size before downloading if Content-Length header is present @@ -229,7 +231,9 @@ def _parse_remote_file(fields: Mapping[str, object]) -> _RemoteFile | None: def _parse_remote_source(fields: Mapping[str, object]) -> _RemoteSource | None: - source: Final = _as_mapping(fields.get("source")) if fields.get("type") in _ANTHROPIC_MEDIA_BLOCK_TYPES else None + part_type: Final = fields.get("type") + is_media_block: Final = isinstance(part_type, str) and part_type in _ANTHROPIC_MEDIA_BLOCK_TYPES + source: Final = _as_mapping(fields.get("source")) if is_media_block else None url: Final = _remote_url(source.get("url")) if source is not None and source.get("type") == "url" else None return _RemoteSource(fields, source, url) if source is not None and url is not None else None @@ -326,21 +330,31 @@ async def async_convert_urls_to_base64( raise +_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None" +_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop + + async def _fetch_outcome( url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None -) -> "str | litellm.ImageFetchError": +) -> _FetchOutcome: try: return await _fetch_data_url(url, in_flight, timeout) except litellm.ImageFetchError as error: - return error + return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error + + +def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]": + _UNREAD_DOWNLOADS.add(download) + download.add_done_callback(_UNREAD_DOWNLOADS.discard) + return download async def _outcomes_through_first_failure( - downloads: "tuple[asyncio.Task[str | litellm.ImageFetchError], ...]", -) -> "AsyncIterator[str | litellm.ImageFetchError]": + downloads: "tuple[asyncio.Task[_FetchOutcome], ...]", +) -> AsyncIterator[_FetchOutcome]: for download in downloads: - yield await download - if isinstance(download.result(), litellm.ImageFetchError): + yield (outcome := await asyncio.shield(download)) + if isinstance(outcome, litellm.ImageFetchError): return @@ -348,14 +362,12 @@ async def async_fetch_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> "Mapping[str, str | litellm.ImageFetchError]": in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - downloads: Final = tuple(asyncio.create_task(_fetch_outcome(url, in_flight, timeout)) for url in remote_urls) - try: - outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) - finally: - for download in downloads: - download.cancel() - await asyncio.gather(*downloads, return_exceptions=True) - return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) + downloads: Final = tuple( + _kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls + ) + outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) + downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True) + return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None}) def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 7aeebfc1f8b..4efa86ad637 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -85,7 +85,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) -_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = frozenset({"user", "tool"}) +_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool") _NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) _prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( "prefetched_pdf_urls" diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 38ef6a0f4c5..4f2a99177ac 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -10,9 +10,9 @@ import threading from collections.abc import Callable from typing import Final +import httpx import pytest import respx -import httpx from httpx import Request, Response import litellm @@ -1801,29 +1801,28 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_the_downloads_after_it( +async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it( respx_mock: respx.MockRouter, ) -> None: dead_url: Final = "https://files.example.test/dead-first.pdf" slow_url: Final = "https://files.example.test/slow-after-dead.pdf" in_memory_cache.delete_cache(slow_url) slow_download_started: Final = asyncio.Event() - slow_download_stopped: Final = asyncio.Event() + slow_download_released: Final = asyncio.Event() + slow_download_served: Final = asyncio.Event() async def answer_not_found_once_the_slow_download_started(request: Request) -> Response: await asyncio.wait_for(slow_download_started.wait(), timeout=5) return Response(404) - async def serve_pdf_never(request: Request) -> Response: + async def serve_pdf_once_released(request: Request) -> Response: slow_download_started.set() - try: - await asyncio.Event().wait() - finally: - slow_download_stopped.set() + await slow_download_released.wait() + slow_download_served.set() return _serve_pdf(request) respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started) - respx_mock.get(slow_url).mock(side_effect=serve_pdf_never) + respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released) with pytest.raises(litellm.ImageFetchError): await asyncio.wait_for( @@ -1836,8 +1835,59 @@ async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_ ), timeout=5, ) + slow_download_released.set() + await asyncio.wait_for(slow_download_served.wait(), timeout=5) + await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) - assert slow_download_stopped.is_set() + assert in_memory_cache.get_cache(slow_url) is not None + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time( + respx_mock: respx.MockRouter, +) -> None: + pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2)) + downloads: Final = tuple( + respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls + ) + for url in pdf_urls: + in_memory_cache.delete_cache(url) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [ + {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]} + ] + assert [download.call_count for download in downloads] == [2, 2] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", ["system", "assistant", "developer"]) +@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")]) +async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request( + role: str, part_type: object +) -> None: + messages: Final = [ + {"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]}, + {"role": "user", "content": "hi"}, + ] + + sync_request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + async_request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request @pytest.mark.usefixtures("pdf_urls_reachable")