mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(decisions): drop images past a model's limits under drop_params, map OpenRouter perplexity models, normalize state images only
Images past a count or size limit now drop under drop_params, keeping the longest in-order run that fits, and answer 400 without it. Cloudflare clef follows the same rule. OpenRouter perplexity/ models take Perplexity's 8-image limit. Inside state, URL-safe base64 becomes standard base64 with padding and image/jpg goes out as image/jpeg, while the images field keeps the caller's base64 as on main.
This commit is contained in:
mateo-berri
parent
08c5a183a8
commit
f716db54a3
4 changed files with 147 additions and 49 deletions
|
|
@ -2,6 +2,7 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
|
|
@ -621,14 +622,27 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
|
|||
)
|
||||
if not unsupported:
|
||||
return call
|
||||
supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(supported):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=supported))
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)),
|
||||
f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}",
|
||||
)
|
||||
|
||||
|
||||
def _with_images_kept(
|
||||
decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool]
|
||||
) -> DecisionsIRState | DecisionsIRMessages:
|
||||
flags: Final = iter(kept)
|
||||
return with_images_where(decision_input, lambda _image: next(flags))
|
||||
|
||||
|
||||
def _dropped_or_refused(
|
||||
call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str
|
||||
) -> DecisionsCall:
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=kept_input))
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, "
|
||||
f"got {', '.join(unsupported)}"
|
||||
),
|
||||
message=f"{call.custom_llm_provider} {refusal}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
|
|
@ -639,10 +653,10 @@ def _checked_image_count(call: DecisionsCall) -> DecisionsCall:
|
|||
image_count: Final = len(input_images(call.ir_request.input))
|
||||
if max_images is None or image_count <= max_images:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
_with_images_kept(call.ir_request.input, tuple(index < max_images for index in range(image_count))),
|
||||
f"takes at most {max_images} images per request, got {image_count}",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -657,12 +671,8 @@ def _byte_limit(limit: int) -> str:
|
|||
return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)"
|
||||
|
||||
|
||||
def _image_size_error(call: DecisionsCall) -> str | None:
|
||||
sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None:
|
||||
largest: Final = max(sizes, default=0)
|
||||
rules: Final = call.provider_config.image_rules_for(call.model)
|
||||
per_image: Final = rules.max_image_bytes
|
||||
total: Final = rules.max_total_image_bytes
|
||||
if per_image is not None and largest > per_image:
|
||||
return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes"
|
||||
if total is not None and sum(sizes) > total:
|
||||
|
|
@ -670,14 +680,26 @@ def _image_size_error(call: DecisionsCall) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _kept_within_sizes(sizes: Sequence[int], per_image: int | None, total: int | None) -> tuple[bool, ...]:
|
||||
fits: Final = tuple(per_image is None or size <= per_image for size in sizes)
|
||||
spent: Final = accumulate(size if fit else 0 for size, fit in zip(sizes, fits, strict=True))
|
||||
return tuple(
|
||||
fit and (total is None or bytes_so_far <= total) for fit, bytes_so_far in zip(fits, spent, strict=True)
|
||||
)
|
||||
|
||||
|
||||
def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall:
|
||||
error: Final = _image_size_error(call)
|
||||
sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
rules: Final = call.provider_config.image_rules_for(call.model)
|
||||
error: Final = _image_size_error(sizes, rules.max_image_bytes, rules.max_total_image_bytes)
|
||||
if error is None:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} {error}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
_with_images_kept(
|
||||
call.ir_request.input, _kept_within_sizes(sizes, rules.max_image_bytes, rules.max_total_image_bytes)
|
||||
),
|
||||
error,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-
|
|||
_REMOTE_URL_PREFIXES: Final = ("http://", "https://")
|
||||
_DATA_URL_PREFIX: Final = "data:"
|
||||
_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/")
|
||||
_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"})
|
||||
|
||||
_StateItem: TypeAlias = str | Mapping[str, object]
|
||||
|
||||
|
|
@ -261,16 +262,25 @@ def image_media_type(image: DecisionsImage) -> str | None:
|
|||
def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return {"content_type": image_media_type(image), "base64": image.base64}
|
||||
return _data_url(image)
|
||||
media_type: Final = image_media_type(image)
|
||||
if media_type is None:
|
||||
return image
|
||||
return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}"
|
||||
|
||||
|
||||
def _standard_data_url(media_type: str, payload: str) -> str:
|
||||
padding: Final = "=" * (-len(payload) % 4)
|
||||
standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type)
|
||||
return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_base64(payload)}{padding}"
|
||||
|
||||
|
||||
def _data_url(image: DecisionsImage) -> str:
|
||||
media_type: Final = image_media_type(image)
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.base64)}"
|
||||
return _standard_data_url(_media_type(image.content_type), image.base64)
|
||||
media_type: Final = image_media_type(image)
|
||||
if media_type is None:
|
||||
return image
|
||||
return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.partition(',')[2])}"
|
||||
return _standard_data_url(media_type, image.partition(",")[2])
|
||||
|
||||
|
||||
def _image_item(image: DecisionsImage) -> _StateItem:
|
||||
|
|
|
|||
|
|
@ -4,11 +4,16 @@ from typing import Final
|
|||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig
|
||||
from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig
|
||||
from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRResponse
|
||||
|
||||
_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType(
|
||||
{"typesafe": TypeSafeDecisionsConfig(), "cloudflare": CloudflareDecisionsConfig()}
|
||||
{
|
||||
"typesafe": TypeSafeDecisionsConfig(),
|
||||
"cloudflare": CloudflareDecisionsConfig(),
|
||||
"perplexity": PerplexityDecisionsConfig(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2297,7 +2297,7 @@ def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, obje
|
|||
def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught:
|
||||
|
|
@ -2435,6 +2435,10 @@ def _png_object(decoded_bytes: int) -> Mapping[str, object]:
|
|||
return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()}
|
||||
|
||||
|
||||
def _distinct_png_data_urls(count: int) -> tuple[str, ...]:
|
||||
return tuple(f"data:image/png;base64,{base64.b64encode(bytes((index,))).decode()}" for index in range(count))
|
||||
|
||||
|
||||
_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = (
|
||||
((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
|
|
@ -2445,29 +2449,62 @@ _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = (
|
|||
)
|
||||
|
||||
|
||||
_OVERSIZED_IMAGE_IDS: Final = ("data_url_over_4mib", "object_over_4mib", "total_over_8mib")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params"))
|
||||
@pytest.mark.parametrize(
|
||||
("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib")
|
||||
)
|
||||
@pytest.mark.parametrize(("images", "refusal"), _OVERSIZED_IMAGES, ids=_OVERSIZED_IMAGE_IDS)
|
||||
def test_images_over_cloudflares_size_limits_are_refused_before_http(
|
||||
images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter
|
||||
images: Sequence[object], refusal: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught:
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=images,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(
|
||||
("images", "kept_count"),
|
||||
tuple(zip((images for images, _ in _OVERSIZED_IMAGES), (0, 0, 2), strict=True)),
|
||||
ids=_OVERSIZED_IMAGE_IDS,
|
||||
)
|
||||
def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params(
|
||||
images: Sequence[object], kept_count: int, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=images,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content).get("images", []) == list(images[:kept_count])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_images_past_cloudflares_limit_are_dropped_under_drop_params(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
images: Final = _distinct_png_data_urls(5)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=list(images),
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["images"] == list(images[:4])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None:
|
||||
|
|
@ -4124,7 +4161,7 @@ def test_systemone_moves_images_into_state(
|
|||
model=model,
|
||||
state={"ticket": 1234},
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
images=["data:IMAGE/JPEG;name=shot.jpg;base64,A-_A", {"content_type": "image/webp", "base64": "B-_B"}],
|
||||
images=["data:IMAGE/JPG;name=shot.jpg;base64,A-_AB-", {"content_type": "image/webp", "base64": "B-_B"}],
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
|
|
@ -4132,7 +4169,7 @@ def test_systemone_moves_images_into_state(
|
|||
"model": upstream_model,
|
||||
"state": [
|
||||
{"ticket": 1234},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/A"}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/AB+=="}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}},
|
||||
],
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
|
|
@ -4160,17 +4197,19 @@ def test_a_list_state_keeps_its_items_ahead_of_the_images(
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "url", "max_images"),
|
||||
(
|
||||
("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8),
|
||||
("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4),
|
||||
),
|
||||
ids=("perplexity", "openrouter-clef"),
|
||||
_STATE_IMAGE_LIMITS: Final = (
|
||||
("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8),
|
||||
("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4),
|
||||
("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions", 8),
|
||||
)
|
||||
_STATE_IMAGE_LIMIT_IDS: Final = ("perplexity", "openrouter-clef", "openrouter-perplexity")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS)
|
||||
def test_more_images_than_the_model_takes_are_refused_before_http(
|
||||
model: str, url: str, max_images: int, respx_mock: respx.MockRouter
|
||||
model: str, url: str, max_images: int, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
|
|
@ -4195,6 +4234,28 @@ def test_more_images_than_the_model_takes_are_refused_before_http(
|
|||
assert route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS)
|
||||
def test_images_past_the_models_limit_are_dropped_under_drop_params(
|
||||
model: str, url: str, max_images: int, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
images: Final = _distinct_png_data_urls(max_images + 2)
|
||||
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=list(images),
|
||||
api_key="k",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["state"] == [
|
||||
_SCREEN_TEXT,
|
||||
*({"type": "image_url", "image_url": {"url": image}} for image in images[:max_images]),
|
||||
]
|
||||
|
||||
|
||||
def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue