fix(openai): let PDF downloads after a dead URL finish and retry rate-limited ones one at a time

Cancelling in-flight downloads after a dead PDF URL could abort a proxy worker on uvloop, so the downloads now run to the end in the background. A URL whose up-front download got a 429 downloads again inside the transform one at a time, as on main. A content part whose type is not a string is no longer read as a PDF part, so it reaches the provider instead of raising a TypeError.
This commit is contained in:
mateo-berri 2026-10-31 07:34:26 -07:00
parent b48f2bf1c7
commit 2c5ca7a78e
4 changed files with 92 additions and 28 deletions

View file

@ -328,7 +328,9 @@ class ImageFetchError(BadRequestError):
max_retries: int | None = None,
num_retries: int | None = None,
body: Mapping[str, object] | None = None,
url_status_code: int | None = None,
):
self.url_status_code = url_status_code
super().__init__(
message=message,
model=model,

View file

@ -7,9 +7,10 @@ import base64
import time
from collections.abc import AsyncIterator, Callable, Iterable, Mapping
from dataclasses import dataclass
from http import HTTPStatus
from itertools import chain, takewhile
from types import MappingProxyType
from typing import Final
from typing import Final, TypeAlias
import httpx
from httpx import Response
@ -39,7 +40,8 @@ def _header_value(response: Response, name: str) -> str | None:
def _process_image_response(response: Response, url: str) -> str:
if response.status_code != 200:
raise litellm.ImageFetchError(
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}",
url_status_code=response.status_code,
)
# Check size before downloading if Content-Length header is present
@ -229,7 +231,9 @@ def _parse_remote_file(fields: Mapping[str, object]) -> _RemoteFile | None:
def _parse_remote_source(fields: Mapping[str, object]) -> _RemoteSource | None:
source: Final = _as_mapping(fields.get("source")) if fields.get("type") in _ANTHROPIC_MEDIA_BLOCK_TYPES else None
part_type: Final = fields.get("type")
is_media_block: Final = isinstance(part_type, str) and part_type in _ANTHROPIC_MEDIA_BLOCK_TYPES
source: Final = _as_mapping(fields.get("source")) if is_media_block else None
url: Final = _remote_url(source.get("url")) if source is not None and source.get("type") == "url" else None
return _RemoteSource(fields, source, url) if source is not None and url is not None else None
@ -326,21 +330,31 @@ async def async_convert_urls_to_base64(
raise
_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None"
_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop
async def _fetch_outcome(
url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None
) -> "str | litellm.ImageFetchError":
) -> _FetchOutcome:
try:
return await _fetch_data_url(url, in_flight, timeout)
except litellm.ImageFetchError as error:
return error
return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error
def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]":
_UNREAD_DOWNLOADS.add(download)
download.add_done_callback(_UNREAD_DOWNLOADS.discard)
return download
async def _outcomes_through_first_failure(
downloads: "tuple[asyncio.Task[str | litellm.ImageFetchError], ...]",
) -> "AsyncIterator[str | litellm.ImageFetchError]":
downloads: "tuple[asyncio.Task[_FetchOutcome], ...]",
) -> AsyncIterator[_FetchOutcome]:
for download in downloads:
yield await download
if isinstance(download.result(), litellm.ImageFetchError):
yield (outcome := await asyncio.shield(download))
if isinstance(outcome, litellm.ImageFetchError):
return
@ -348,14 +362,12 @@ async def async_fetch_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> "Mapping[str, str | litellm.ImageFetchError]":
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
downloads: Final = tuple(asyncio.create_task(_fetch_outcome(url, in_flight, timeout)) for url in remote_urls)
try:
outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)])
finally:
for download in downloads:
download.cancel()
await asyncio.gather(*downloads, return_exceptions=True)
return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True)))
downloads: Final = tuple(
_kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls
)
outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)])
downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True)
return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None})
def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]:

View file

@ -85,7 +85,7 @@ else:
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = frozenset({"user", "tool"})
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool")
_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({})
_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar(
"prefetched_pdf_urls"

View file

@ -10,9 +10,9 @@ import threading
from collections.abc import Callable
from typing import Final
import httpx
import pytest
import respx
import httpx
from httpx import Request, Response
import litellm
@ -1801,29 +1801,28 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_
@pytest.mark.usefixtures("pdf_urls_reachable")
@pytest.mark.respx(assert_all_called=False)
@pytest.mark.asyncio
async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_the_downloads_after_it(
async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it(
respx_mock: respx.MockRouter,
) -> None:
dead_url: Final = "https://files.example.test/dead-first.pdf"
slow_url: Final = "https://files.example.test/slow-after-dead.pdf"
in_memory_cache.delete_cache(slow_url)
slow_download_started: Final = asyncio.Event()
slow_download_stopped: Final = asyncio.Event()
slow_download_released: Final = asyncio.Event()
slow_download_served: Final = asyncio.Event()
async def answer_not_found_once_the_slow_download_started(request: Request) -> Response:
await asyncio.wait_for(slow_download_started.wait(), timeout=5)
return Response(404)
async def serve_pdf_never(request: Request) -> Response:
async def serve_pdf_once_released(request: Request) -> Response:
slow_download_started.set()
try:
await asyncio.Event().wait()
finally:
slow_download_stopped.set()
await slow_download_released.wait()
slow_download_served.set()
return _serve_pdf(request)
respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started)
respx_mock.get(slow_url).mock(side_effect=serve_pdf_never)
respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released)
with pytest.raises(litellm.ImageFetchError):
await asyncio.wait_for(
@ -1836,8 +1835,59 @@ async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_
),
timeout=5,
)
slow_download_released.set()
await asyncio.wait_for(slow_download_served.wait(), timeout=5)
await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5)
assert slow_download_stopped.is_set()
assert in_memory_cache.get_cache(slow_url) is not None
@pytest.mark.usefixtures("pdf_urls_reachable")
@pytest.mark.respx(assert_all_called=True)
@pytest.mark.asyncio
async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time(
respx_mock: respx.MockRouter,
) -> None:
pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2))
downloads: Final = tuple(
respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls
)
for url in pdf_urls:
in_memory_cache.delete_cache(url)
request: Final = await OpenAIGPTConfig().async_transform_request(
model="gpt-4.1",
messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}],
optional_params={},
litellm_params={},
headers={},
)
assert request["messages"] == [
{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]}
]
assert [download.call_count for download in downloads] == [2, 2]
@pytest.mark.asyncio
@pytest.mark.parametrize("role", ["system", "assistant", "developer"])
@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")])
async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request(
role: str, part_type: object
) -> None:
messages: Final = [
{"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]},
{"role": "user", "content": "hi"},
]
sync_request: Final = OpenAIGPTConfig().transform_request(
model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
)
async_request: Final = await OpenAIGPTConfig().async_transform_request(
model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
)
assert async_request == sync_request
@pytest.mark.usefixtures("pdf_urls_reachable")