mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(openai): download a request's PDF URLs one at a time and stop at the first dead one
This commit is contained in:
parent
2c5ca7a78e
commit
8b9f55a4b0
3 changed files with 47 additions and 106 deletions
|
|
@ -328,9 +328,7 @@ class ImageFetchError(BadRequestError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
url_status_code: int | None = None,
|
||||
):
|
||||
self.url_status_code = url_status_code
|
||||
super().__init__(
|
||||
message=message,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,9 @@ import base64
|
|||
import time
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from itertools import chain, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
|
|
@ -40,8 +39,7 @@ def _header_value(response: Response, name: str) -> str | None:
|
|||
def _process_image_response(response: Response, url: str) -> str:
|
||||
if response.status_code != 200:
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}",
|
||||
url_status_code=response.status_code,
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
|
||||
)
|
||||
|
||||
# Check size before downloading if Content-Length header is present
|
||||
|
|
@ -330,30 +328,18 @@ async def async_convert_urls_to_base64(
|
|||
raise
|
||||
|
||||
|
||||
_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None"
|
||||
_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop
|
||||
|
||||
|
||||
async def _fetch_outcome(
|
||||
url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None
|
||||
) -> _FetchOutcome:
|
||||
async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError":
|
||||
try:
|
||||
return await _fetch_data_url(url, in_flight, timeout)
|
||||
return await async_convert_url_to_base64(url, timeout=timeout)
|
||||
except litellm.ImageFetchError as error:
|
||||
return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error
|
||||
return error
|
||||
|
||||
|
||||
def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]":
|
||||
_UNREAD_DOWNLOADS.add(download)
|
||||
download.add_done_callback(_UNREAD_DOWNLOADS.discard)
|
||||
return download
|
||||
|
||||
|
||||
async def _outcomes_through_first_failure(
|
||||
downloads: "tuple[asyncio.Task[_FetchOutcome], ...]",
|
||||
) -> AsyncIterator[_FetchOutcome]:
|
||||
for download in downloads:
|
||||
yield (outcome := await asyncio.shield(download))
|
||||
async def _downloads_through_first_failure(
|
||||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None
|
||||
) -> "AsyncIterator[str | litellm.ImageFetchError]":
|
||||
for url in remote_urls:
|
||||
yield (outcome := await _download_outcome(url, timeout))
|
||||
if isinstance(outcome, litellm.ImageFetchError):
|
||||
return
|
||||
|
||||
|
|
@ -361,13 +347,8 @@ async def _outcomes_through_first_failure(
|
|||
async def async_fetch_urls_to_base64(
|
||||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
|
||||
) -> "Mapping[str, str | litellm.ImageFetchError]":
|
||||
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
|
||||
downloads: Final = tuple(
|
||||
_kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls
|
||||
)
|
||||
outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)])
|
||||
downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True)
|
||||
return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None})
|
||||
outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)])
|
||||
return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True)))
|
||||
|
||||
|
||||
def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation
|
|||
import asyncio
|
||||
import base64
|
||||
import copy
|
||||
import itertools
|
||||
import json
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
|
|
@ -1475,25 +1474,24 @@ async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cac
|
|||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_the_same_time(
|
||||
async def test_async_transform_request_downloads_pdf_urls_one_at_a_time_for_a_host_that_refuses_bursts(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_urls: Final = tuple(f"https://files.example.test/together-{index}.pdf" for index in range(3))
|
||||
pdf_urls: Final = tuple(f"https://files.example.test/in-turn-{index}.pdf" for index in range(3))
|
||||
for pdf_url in pdf_urls:
|
||||
in_memory_cache.delete_cache(pdf_url)
|
||||
started_downloads: Final = itertools.count(1)
|
||||
every_download_started: Final = asyncio.Event()
|
||||
download_running: Final = asyncio.Lock()
|
||||
|
||||
async def serve_pdf_once_every_download_started(request: Request) -> Response:
|
||||
if next(started_downloads) == len(pdf_urls):
|
||||
every_download_started.set()
|
||||
try:
|
||||
await asyncio.wait_for(every_download_started.wait(), timeout=5)
|
||||
except TimeoutError as error:
|
||||
raise litellm.ImageFetchError("the PDF URLs were downloaded one after another") from error
|
||||
return await _serve_pdf_to_async_clients_only(request)
|
||||
async def refuse_a_second_download_at_once(request: Request) -> Response:
|
||||
if download_running.locked():
|
||||
return Response(429)
|
||||
async with download_running:
|
||||
await asyncio.sleep(0.05)
|
||||
return await _serve_pdf_to_async_clients_only(request)
|
||||
|
||||
respx_mock.route(method="GET", host="files.example.test").mock(side_effect=serve_pdf_once_every_download_started)
|
||||
downloads: Final = respx_mock.route(method="GET", host="files.example.test").mock(
|
||||
side_effect=refuse_a_second_download_at_once
|
||||
)
|
||||
|
||||
request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
|
|
@ -1504,6 +1502,7 @@ async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_
|
|||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}]
|
||||
assert downloads.call_count == len(pdf_urls)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
|
|
@ -1702,6 +1701,10 @@ def _answer_not_found(request: Request) -> Response:
|
|||
return Response(404)
|
||||
|
||||
|
||||
def _answer_too_many_requests(request: Request) -> Response:
|
||||
return Response(429)
|
||||
|
||||
|
||||
def _refuse_connection(request: Request) -> Response:
|
||||
raise httpx.ConnectError("connection refused", request=request)
|
||||
|
||||
|
|
@ -1771,7 +1774,12 @@ async def test_async_transform_request_matches_the_sync_request_downloads_and_ca
|
|||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"dead_url_answer", [pytest.param(_answer_not_found, id="not_found"), pytest.param(_refuse_connection, id="refused")]
|
||||
"dead_url_answer",
|
||||
[
|
||||
pytest.param(_answer_not_found, id="not_found"),
|
||||
pytest.param(_answer_too_many_requests, id="rate_limited"),
|
||||
pytest.param(_refuse_connection, id="refused"),
|
||||
],
|
||||
)
|
||||
async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count(
|
||||
respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response]
|
||||
|
|
@ -1801,72 +1809,26 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_
|
|||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=False)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it(
|
||||
async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
dead_url: Final = "https://files.example.test/dead-first.pdf"
|
||||
slow_url: Final = "https://files.example.test/slow-after-dead.pdf"
|
||||
in_memory_cache.delete_cache(slow_url)
|
||||
slow_download_started: Final = asyncio.Event()
|
||||
slow_download_released: Final = asyncio.Event()
|
||||
slow_download_served: Final = asyncio.Event()
|
||||
|
||||
async def answer_not_found_once_the_slow_download_started(request: Request) -> Response:
|
||||
await asyncio.wait_for(slow_download_started.wait(), timeout=5)
|
||||
return Response(404)
|
||||
|
||||
async def serve_pdf_once_released(request: Request) -> Response:
|
||||
slow_download_started.set()
|
||||
await slow_download_released.wait()
|
||||
slow_download_served.set()
|
||||
return _serve_pdf(request)
|
||||
|
||||
respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started)
|
||||
respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released)
|
||||
later_url: Final = "https://files.example.test/after-dead.pdf"
|
||||
in_memory_cache.delete_cache(later_url)
|
||||
respx_mock.get(dead_url).mock(return_value=Response(404))
|
||||
later: Final = respx_mock.get(later_url).mock(side_effect=_serve_pdf_to_async_clients_only)
|
||||
|
||||
with pytest.raises(litellm.ImageFetchError):
|
||||
await asyncio.wait_for(
|
||||
OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(slow_url)]}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
),
|
||||
timeout=5,
|
||||
await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(later_url)]}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
slow_download_released.set()
|
||||
await asyncio.wait_for(slow_download_served.wait(), timeout=5)
|
||||
await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5)
|
||||
|
||||
assert in_memory_cache.get_cache(slow_url) is not None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2))
|
||||
downloads: Final = tuple(
|
||||
respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls
|
||||
)
|
||||
for url in pdf_urls:
|
||||
in_memory_cache.delete_cache(url)
|
||||
|
||||
request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [
|
||||
{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]}
|
||||
]
|
||||
assert [download.call_count for download in downloads] == [2, 2]
|
||||
assert later.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue