diff --git a/litellm/exceptions.py b/litellm/exceptions.py index efd8a1323fe..12425c45b01 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -328,9 +328,7 @@ class ImageFetchError(BadRequestError): max_retries: int | None = None, num_retries: int | None = None, body: Mapping[str, object] | None = None, - url_status_code: int | None = None, ): - self.url_status_code = url_status_code super().__init__( message=message, model=model, diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 6385db9a742..e49afbb07b9 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -7,10 +7,9 @@ import base64 import time from collections.abc import AsyncIterator, Callable, Iterable, Mapping from dataclasses import dataclass -from http import HTTPStatus from itertools import chain, takewhile from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final import httpx from httpx import Response @@ -40,8 +39,7 @@ def _header_value(response: Response, name: str) -> str | None: def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: raise litellm.ImageFetchError( - f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}", - url_status_code=response.status_code, + f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}" ) # Check size before downloading if Content-Length header is present @@ -330,30 +328,18 @@ async def async_convert_urls_to_base64( raise -_FetchOutcome: TypeAlias = "str | litellm.ImageFetchError | None" -_UNREAD_DOWNLOADS: Final[set[asyncio.Task[_FetchOutcome]]] = set() # mutable-ok: cancelling a download aborts uvloop - - -async def _fetch_outcome( - url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None -) -> _FetchOutcome: +async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError": try: - return await _fetch_data_url(url, in_flight, timeout) + return await async_convert_url_to_base64(url, timeout=timeout) except litellm.ImageFetchError as error: - return None if error.url_status_code == HTTPStatus.TOO_MANY_REQUESTS else error + return error -def _kept_until_done(download: "asyncio.Task[_FetchOutcome]") -> "asyncio.Task[_FetchOutcome]": - _UNREAD_DOWNLOADS.add(download) - download.add_done_callback(_UNREAD_DOWNLOADS.discard) - return download - - -async def _outcomes_through_first_failure( - downloads: "tuple[asyncio.Task[_FetchOutcome], ...]", -) -> AsyncIterator[_FetchOutcome]: - for download in downloads: - yield (outcome := await asyncio.shield(download)) +async def _downloads_through_first_failure( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None +) -> "AsyncIterator[str | litellm.ImageFetchError]": + for url in remote_urls: + yield (outcome := await _download_outcome(url, timeout)) if isinstance(outcome, litellm.ImageFetchError): return @@ -361,13 +347,8 @@ async def _outcomes_through_first_failure( async def async_fetch_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> "Mapping[str, str | litellm.ImageFetchError]": - in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - downloads: Final = tuple( - _kept_until_done(asyncio.create_task(_fetch_outcome(url, in_flight, timeout))) for url in remote_urls - ) - outcomes: Final = tuple([outcome async for outcome in _outcomes_through_first_failure(downloads)]) - downloaded: Final = zip(remote_urls[: len(outcomes)], outcomes, strict=True) - return MappingProxyType({url: outcome for url, outcome in downloaded if outcome is not None}) + outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)]) + return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) def _remote_urls_in_parts(parts: Iterable[object], should_inline: Callable[[RemoteMedia], bool]) -> tuple[str, ...]: diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 4f2a99177ac..46c99df463b 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -4,7 +4,6 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation import asyncio import base64 import copy -import itertools import json import threading from collections.abc import Callable @@ -1475,25 +1474,24 @@ async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cac @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio -async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_the_same_time( +async def test_async_transform_request_downloads_pdf_urls_one_at_a_time_for_a_host_that_refuses_bursts( respx_mock: respx.MockRouter, ) -> None: - pdf_urls: Final = tuple(f"https://files.example.test/together-{index}.pdf" for index in range(3)) + pdf_urls: Final = tuple(f"https://files.example.test/in-turn-{index}.pdf" for index in range(3)) for pdf_url in pdf_urls: in_memory_cache.delete_cache(pdf_url) - started_downloads: Final = itertools.count(1) - every_download_started: Final = asyncio.Event() + download_running: Final = asyncio.Lock() - async def serve_pdf_once_every_download_started(request: Request) -> Response: - if next(started_downloads) == len(pdf_urls): - every_download_started.set() - try: - await asyncio.wait_for(every_download_started.wait(), timeout=5) - except TimeoutError as error: - raise litellm.ImageFetchError("the PDF URLs were downloaded one after another") from error - return await _serve_pdf_to_async_clients_only(request) + async def refuse_a_second_download_at_once(request: Request) -> Response: + if download_running.locked(): + return Response(429) + async with download_running: + await asyncio.sleep(0.05) + return await _serve_pdf_to_async_clients_only(request) - respx_mock.route(method="GET", host="files.example.test").mock(side_effect=serve_pdf_once_every_download_started) + downloads: Final = respx_mock.route(method="GET", host="files.example.test").mock( + side_effect=refuse_a_second_download_at_once + ) request: Final = await OpenAIGPTConfig().async_transform_request( model="gpt-4.1", @@ -1504,6 +1502,7 @@ async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_ ) assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}] + assert downloads.call_count == len(pdf_urls) @pytest.mark.usefixtures("pdf_urls_reachable") @@ -1702,6 +1701,10 @@ def _answer_not_found(request: Request) -> Response: return Response(404) +def _answer_too_many_requests(request: Request) -> Response: + return Response(429) + + def _refuse_connection(request: Request) -> Response: raise httpx.ConnectError("connection refused", request=request) @@ -1771,7 +1774,12 @@ async def test_async_transform_request_matches_the_sync_request_downloads_and_ca @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio @pytest.mark.parametrize( - "dead_url_answer", [pytest.param(_answer_not_found, id="not_found"), pytest.param(_refuse_connection, id="refused")] + "dead_url_answer", + [ + pytest.param(_answer_not_found, id="not_found"), + pytest.param(_answer_too_many_requests, id="rate_limited"), + pytest.param(_refuse_connection, id="refused"), + ], ) async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count( respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response] @@ -1801,72 +1809,26 @@ async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_ @pytest.mark.usefixtures("pdf_urls_reachable") @pytest.mark.respx(assert_all_called=False) @pytest.mark.asyncio -async def test_async_transform_request_fails_a_dead_pdf_url_without_waiting_for_or_cancelling_the_downloads_after_it( +async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it( respx_mock: respx.MockRouter, ) -> None: dead_url: Final = "https://files.example.test/dead-first.pdf" - slow_url: Final = "https://files.example.test/slow-after-dead.pdf" - in_memory_cache.delete_cache(slow_url) - slow_download_started: Final = asyncio.Event() - slow_download_released: Final = asyncio.Event() - slow_download_served: Final = asyncio.Event() - - async def answer_not_found_once_the_slow_download_started(request: Request) -> Response: - await asyncio.wait_for(slow_download_started.wait(), timeout=5) - return Response(404) - - async def serve_pdf_once_released(request: Request) -> Response: - slow_download_started.set() - await slow_download_released.wait() - slow_download_served.set() - return _serve_pdf(request) - - respx_mock.get(dead_url).mock(side_effect=answer_not_found_once_the_slow_download_started) - respx_mock.get(slow_url).mock(side_effect=serve_pdf_once_released) + later_url: Final = "https://files.example.test/after-dead.pdf" + in_memory_cache.delete_cache(later_url) + respx_mock.get(dead_url).mock(return_value=Response(404)) + later: Final = respx_mock.get(later_url).mock(side_effect=_serve_pdf_to_async_clients_only) with pytest.raises(litellm.ImageFetchError): - await asyncio.wait_for( - OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(slow_url)]}], - optional_params={}, - litellm_params={}, - headers={}, - ), - timeout=5, + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(dead_url), _pdf_url_part(later_url)]}], + optional_params={}, + litellm_params={}, + headers={}, ) - slow_download_released.set() - await asyncio.wait_for(slow_download_served.wait(), timeout=5) await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) - assert in_memory_cache.get_cache(slow_url) is not None - - -@pytest.mark.usefixtures("pdf_urls_reachable") -@pytest.mark.respx(assert_all_called=True) -@pytest.mark.asyncio -async def test_async_transform_request_downloads_rate_limited_pdf_urls_again_one_at_a_time( - respx_mock: respx.MockRouter, -) -> None: - pdf_urls: Final = tuple(f"https://files.example.test/rate-limited-{index}.pdf" for index in range(2)) - downloads: Final = tuple( - respx_mock.get(url).mock(side_effect=[Response(429), _serve_pdf(Request("GET", url))]) for url in pdf_urls - ) - for url in pdf_urls: - in_memory_cache.delete_cache(url) - - request: Final = await OpenAIGPTConfig().async_transform_request( - model="gpt-4.1", - messages=[{"role": "user", "content": [_SUMMARIZE, *(_pdf_url_part(url) for url in pdf_urls)]}], - optional_params={}, - litellm_params={}, - headers={}, - ) - - assert request["messages"] == [ - {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART, _DOWNLOADED_PDF_PART]} - ] - assert [download.call_count for download in downloads] == [2, 2] + assert later.call_count == 0 @pytest.mark.asyncio