mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(openai): let PDF downloads after a dead URL finish and retry rate-limited ones one at a time
Cancelling in-flight downloads after a dead PDF URL could abort a proxy worker on uvloop, so the downloads now run to the end in the background. A URL whose up-front download got a 429 downloads again inside the transform one at a time, as on main. A content part whose type is not a string is no longer read as a PDF part, so it reaches the provider instead of raising a TypeError.
This commit is contained in:
parent
b48f2bf1c7
commit
2c5ca7a78e
4 changed files with 92 additions and 28 deletions
|
|
@ -328,7 +328,9 @@ class ImageFetchError(BadRequestError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
url_status_code: int | None = None,
|
||||
):
|
||||
self.url_status_code = url_status_code
|
||||
super().__init__(
|
||||
message=message,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@ import base64
|
|||
import time
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from itertools import chain, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
|
|
@ -39,7 +40,8 @@ def _header_value(response: Response, name: str) -> str | None:
|
|||
def _process_image_response(response: Response, url: str) -> str:
|
||||
if response.status_code != 200:
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}",
|
||||
url_status_code=response.status_code,
|
||||
)
|
||||
|
||||
# Check size before downloading if Content-Length header is present
|
||||
|
|
@ -229,7 +231,9 @@ def _parse_remote_file(fields: Mapping[str, object]) -> _RemoteFile | None:
|
|||
|
||||
|
||||
def _parse_remote_source(fields: Mapping[str, object]) -> _RemoteSource | None:
|
||||
source: Final = _as_mapping(fields.get("source")) if fields.get("type") in _ANTHROPIC_MEDIA_BLOCK_TYPES else None
|
||||
part_type: Final = fields.get("type")
|
||||
is_media_block: Final = isinstance(part_type, str) and part_type in _ANTHROPIC_MEDIA_BLOCK_TYPES
|
||||
source: Final = _as_mapping(fields.get("source")) if is_media_block else None
|
||||
url: Final = _remote_url(source.get("url")) if source is not None and source.get("type") == "url" else None
|
||||
return _RemoteSource(fields, source, url) if source is not None and url is not None else None
|
||||
|
||||
|
|
@ -326,21 +330,31 @@ async def async_convert_urls_to_base64(
|
|||
raise
|
||||
|
||||
|
||||
_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None"
|
||||
_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop
|
||||
|
||||
|
||||
async def _fetch_outcome(
|
||||
url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None
|
||||
) -> "str | litellm.ImageFetchError":
|
||||
) -> _FetchOutcome:
|
||||
try:
|
||||
return await _fetch_data_url(url, in_flight, timeout)
|
||||
except litellm.ImageFetchError as error:
|
||||
return error
|
||||
return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error
|
||||
|
||||
|
||||
def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]":
|
||||
_UNREAD_DOWNLOADS.add(download)
|
||||
download.add_done_callback(_UNREAD_DOWNLOADS.discard)
|
||||
return download
|
||||
|
||||
|
||||
async def _outcomes_through_first_failure(
|
||||
downloads: "tuple[asyncio.Task[str | litellm.ImageFetchError], ...]",
|
||||
) -> "AsyncIterator[str | litellm.ImageFetchError]":
|
||||
downloads: "tuple[asyncio.Task[_FetchOutcome], ...]",
|
||||
) -> AsyncIterator[_FetchOutcome]:
|
||||
for download in downloads:
|
||||
yield await download
|
||||
if isinstance(download.result(), litellm.ImageFetchError):
|
||||
yield (outcome := await asyncio.shield(download))
|
||||
if isinstance(outcome, litellm.ImageFetchError):
|
||||
return
|
||||
|
||||
|
||||
|
|
@ -348,14 +362,12 @@ async def async_fetch_urls_to_base64(
|
|||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
|
||||
) -> "Mapping[str, str | litellm.ImageFetchError]":
|
||||
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
|
||||
downloads: Final = tuple(asyncio.create_task(_fetch_outcome(url, in_flight, timeout)) for url in remote_urls)
|
||||
try:
|
||||
outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)])
|
||||
finally:
|
||||
for download in downloads:
|
||||
download.cancel()
|
||||
await asyncio.gather(*downloads, return_exceptions=True)
|
||||
return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True)))
|
||||
downloads: Final = tuple(
|
||||
_kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls
|
||||
)
|
||||
outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)])
|
||||
downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True)
|
||||
return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None})
|
||||
|
||||
|
||||
def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ else:
|
|||
|
||||
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = frozenset({"user", "tool"})
|
||||
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool")
|
||||
_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({})
|
||||
_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar(
|
||||
"prefetched_pdf_urls"
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ import threading
|
|||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
import httpx
|
||||
from httpx import Request, Response
|
||||
|
||||
import litellm
|
||||
|
|
@ -1801,29 +1801,28 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_
|
|||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=False)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_the_downloads_after_it(
|
||||
async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
dead_url: Final = "https://files.example.test/dead-first.pdf"
|
||||
slow_url: Final = "https://files.example.test/slow-after-dead.pdf"
|
||||
in_memory_cache.delete_cache(slow_url)
|
||||
slow_download_started: Final = asyncio.Event()
|
||||
slow_download_stopped: Final = asyncio.Event()
|
||||
slow_download_released: Final = asyncio.Event()
|
||||
slow_download_served: Final = asyncio.Event()
|
||||
|
||||
async def answer_not_found_once_the_slow_download_started(request: Request) -> Response:
|
||||
await asyncio.wait_for(slow_download_started.wait(), timeout=5)
|
||||
return Response(404)
|
||||
|
||||
async def serve_pdf_never(request: Request) -> Response:
|
||||
async def serve_pdf_once_released(request: Request) -> Response:
|
||||
slow_download_started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
slow_download_stopped.set()
|
||||
await slow_download_released.wait()
|
||||
slow_download_served.set()
|
||||
return _serve_pdf(request)
|
||||
|
||||
respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started)
|
||||
respx_mock.get(slow_url).mock(side_effect=serve_pdf_never)
|
||||
respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released)
|
||||
|
||||
with pytest.raises(litellm.ImageFetchError):
|
||||
await asyncio.wait_for(
|
||||
|
|
@ -1836,8 +1835,59 @@ async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_
|
|||
),
|
||||
timeout=5,
|
||||
)
|
||||
slow_download_released.set()
|
||||
await asyncio.wait_for(slow_download_served.wait(), timeout=5)
|
||||
await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5)
|
||||
|
||||
assert slow_download_stopped.is_set()
|
||||
assert in_memory_cache.get_cache(slow_url) is not None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2))
|
||||
downloads: Final = tuple(
|
||||
respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls
|
||||
)
|
||||
for url in pdf_urls:
|
||||
in_memory_cache.delete_cache(url)
|
||||
|
||||
request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [
|
||||
{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]}
|
||||
]
|
||||
assert [download.call_count for download in downloads] == [2, 2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", ["system", "assistant", "developer"])
|
||||
@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")])
|
||||
async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request(
|
||||
role: str, part_type: object
|
||||
) -> None:
|
||||
messages: Final = [
|
||||
{"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]},
|
||||
{"role": "user", "content": "hi"},
|
||||
]
|
||||
|
||||
sync_request: Final = OpenAIGPTConfig().transform_request(
|
||||
model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
async_request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert async_request == sync_request
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue