diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 63bd1222d3f..feebd664246 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -2,6 +2,7 @@ import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial +from itertools import accumulate from types import MappingProxyType from typing import Final, Literal, TypeAlias @@ -621,14 +622,27 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall: ) if not unsupported: return call - supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)) - if _drops_params(call.litellm_params) and not _leaves_no_input(supported): - return replace(call, ir_request=replace(call.ir_request, input=supported)) + return _dropped_or_refused( + call, + with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)), + f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}", + ) + + +def _with_images_kept( + decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool] +) -> DecisionsIRState | DecisionsIRMessages: + flags: Final = iter(kept) + return with_images_where(decision_input, lambda _image: next(flags)) + + +def _dropped_or_refused( + call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str +) -> DecisionsCall: + if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input): + return replace(call, ir_request=replace(call.ir_request, input=kept_input)) raise litellm.BadRequestError( - message=( - f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, " - f"got {', '.join(unsupported)}" - ), + message=f"{call.custom_llm_provider} {refusal}", model=call.requested_model, llm_provider=call.custom_llm_provider, ) @@ -639,10 +653,10 @@ def _checked_image_count(call: DecisionsCall) -> DecisionsCall: image_count: Final = len(input_images(call.ir_request.input)) if max_images is None or image_count <= max_images: return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, + return _dropped_or_refused( + call, + _with_images_kept(call.ir_request.input, tuple(index < max_images for index in range(image_count))), + f"takes at most {max_images} images per request, got {image_count}", ) @@ -657,12 +671,8 @@ def _byte_limit(limit: int) -> str: return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)" -def _image_size_error(call: DecisionsCall) -> str | None: - sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) +def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None: largest: Final = max(sizes, default=0) - rules: Final = call.provider_config.image_rules_for(call.model) - per_image: Final = rules.max_image_bytes - total: Final = rules.max_total_image_bytes if per_image is not None and largest > per_image: return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes" if total is not None and sum(sizes) > total: @@ -670,14 +680,26 @@ def _image_size_error(call: DecisionsCall) -> str | None: return None +def _kept_within_sizes(sizes: Sequence[int], per_image: int | None, total: int | None) -> tuple[bool, ...]: + fits: Final = tuple(per_image is None or size <= per_image for size in sizes) + spent: Final = accumulate(size if fit else 0 for size, fit in zip(sizes, fits, strict=True)) + return tuple( + fit and (total is None or bytes_so_far <= total) for fit, bytes_so_far in zip(fits, spent, strict=True) + ) + + def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall: - error: Final = _image_size_error(call) + sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) + rules: Final = call.provider_config.image_rules_for(call.model) + error: Final = _image_size_error(sizes, rules.max_image_bytes, rules.max_total_image_bytes) if error is None: return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} {error}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, + return _dropped_or_refused( + call, + _with_images_kept( + call.ir_request.input, _kept_within_sizes(sizes, rules.max_image_bytes, rules.max_total_image_bytes) + ), + error, ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 50dacadbe07..7edc96bf525 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -66,6 +66,7 @@ _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content- _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" _URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") +_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"}) _StateItem: TypeAlias = str | Mapping[str, object] @@ -261,16 +262,25 @@ def image_media_type(image: DecisionsImage) -> str | None: def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]: if isinstance(image, DecisionsBase64Image): return {"content_type": image_media_type(image), "base64": image.base64} - return _data_url(image) + media_type: Final = image_media_type(image) + if media_type is None: + return image + return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}" + + +def _standard_data_url(media_type: str, payload: str) -> str: + padding: Final = "=" * (-len(payload) % 4) + standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type) + return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_base64(payload)}{padding}" def _data_url(image: DecisionsImage) -> str: - media_type: Final = image_media_type(image) if isinstance(image, DecisionsBase64Image): - return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.base64)}" + return _standard_data_url(_media_type(image.content_type), image.base64) + media_type: Final = image_media_type(image) if media_type is None: return image - return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.partition(',')[2])}" + return _standard_data_url(media_type, image.partition(",")[2]) def _image_item(image: DecisionsImage) -> _StateItem: diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index b1af997920b..49d42036cd7 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -4,11 +4,16 @@ from typing import Final from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig +from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig from litellm.types.decisions import DecisionsIRResponse _ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( - {"typesafe": TypeSafeDecisionsConfig(), "cloudflare": CloudflareDecisionsConfig()} + { + "typesafe": TypeSafeDecisionsConfig(), + "cloudflare": CloudflareDecisionsConfig(), + "perplexity": PerplexityDecisionsConfig(), + } ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 6df93598e80..a9f27fe176c 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -2297,7 +2297,7 @@ def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, obje def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http( api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(litellm, "drop_params", True) + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: @@ -2435,6 +2435,10 @@ def _png_object(decoded_bytes: int) -> Mapping[str, object]: return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} +def _distinct_png_data_urls(count: int) -> tuple[str, ...]: + return tuple(f"data:image/png;base64,{base64.b64encode(bytes((index,))).decode()}" for index in range(count)) + + _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( ((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), ((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), @@ -2445,29 +2449,62 @@ _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( ) +_OVERSIZED_IMAGE_IDS: Final = ("data_url_over_4mib", "object_over_4mib", "total_over_8mib") + + @pytest.mark.usefixtures("cloudflare_account") -@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) -@pytest.mark.parametrize( - ("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib") -) +@pytest.mark.parametrize(("images", "refusal"), _OVERSIZED_IMAGES, ids=_OVERSIZED_IMAGE_IDS) def test_images_over_cloudflares_size_limits_are_refused_before_http( - images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter + images: Sequence[object], refusal: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught: - litellm.systemone( - model="cloudflare/clef", - state=_SCREEN_TEXT, - questions=_SYSTEMONE_PREDICATE, - images=images, - drop_params=drop_params, - ) + litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images) assert caught.value.status_code == 400 assert not route.called +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize( + ("images", "kept_count"), + tuple(zip((images for images, _ in _OVERSIZED_IMAGES), (0, 0, 2), strict=True)), + ids=_OVERSIZED_IMAGE_IDS, +) +def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params( + images: Sequence[object], kept_count: int, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=images, + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content).get("images", []) == list(images[:kept_count]) + + +@pytest.mark.usefixtures("cloudflare_account") +def test_images_past_cloudflares_limit_are_dropped_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(5) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["images"] == list(images[:4]) + + @pytest.mark.asyncio @pytest.mark.usefixtures("cloudflare_account") async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None: @@ -4124,7 +4161,7 @@ def test_systemone_moves_images_into_state( model=model, state={"ticket": 1234}, questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - images=["data:IMAGE/JPEG;name=shot.jpg;base64,A-_A", {"content_type": "image/webp", "base64": "B-_B"}], + images=["data:IMAGE/JPG;name=shot.jpg;base64,A-_AB-", {"content_type": "image/webp", "base64": "B-_B"}], api_key="caller-key", ) @@ -4132,7 +4169,7 @@ def test_systemone_moves_images_into_state( "model": upstream_model, "state": [ {"ticket": 1234}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/A"}}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/AB+=="}}, {"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}}, ], "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -4160,17 +4197,19 @@ def test_a_list_state_keeps_its_items_ahead_of_the_images( ] -@pytest.mark.parametrize( - ("model", "url", "max_images"), - ( - ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8), - ("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4), - ), - ids=("perplexity", "openrouter-clef"), +_STATE_IMAGE_LIMITS: Final = ( + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8), + ("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4), + ("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions", 8), ) +_STATE_IMAGE_LIMIT_IDS: Final = ("perplexity", "openrouter-clef", "openrouter-perplexity") + + +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) def test_more_images_than_the_model_takes_are_refused_before_http( - model: str, url: str, max_images: int, respx_mock: respx.MockRouter + model: str, url: str, max_images: int, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(url).respond(json=_RESPONSE) litellm.systemone( @@ -4195,6 +4234,28 @@ def test_more_images_than_the_model_takes_are_refused_before_http( assert route.call_count == 1 +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) +def test_images_past_the_models_limit_are_dropped_under_drop_params( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(max_images + 2) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + api_key="k", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + *({"type": "image_url", "image_url": {"url": image}} for image in images[:max_images]), + ] + + def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8)