Merge 9c005c96cc into 2d2290067c

litellm/decisions/call.py

@@ -1,7 +1,9 @@
+import base64
 import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
+from itertools import accumulate, islice
 from types import MappingProxyType
 from typing import Annotated, Final, Literal, TypeAlias
 
@@ -18,7 +20,9 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpe
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     async_convert_urls_to_base64,
+    async_convert_urls_to_base64_or_errors,
     convert_url_to_base64,
+    convert_url_to_base64_or_error,
 )
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.decisions.transformation import (
@@ -27,9 +31,11 @@ from litellm.llms.base_llm.decisions.transformation import (
     catalog_mode,
     image_media_type,
     image_param,
+    input_has_no_text,
     input_images,
     ir_to_systemone_response,
     remote_image_urls,
+    standard_base64,
     systemone_request_to_ir,
     with_images_where,
     with_inlined_images,
@@ -87,6 +93,7 @@ _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[
 ] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None)
 _HANDLER: Final = BaseLLMHTTPHandler()
 _BASE64_MARKER: Final = ";base64,"
+_SNIFFABLE_MEDIA_TYPES: Final = frozenset({"image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp"})
 
 
 _IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images"))
@@ -380,6 +387,7 @@ class _MalformedImages:
 def _unsupported_params(
     ir_request: DecisionsIRRequest,
     provider_config: BaseDecisionsConfig,
+    model: str,
     headers: Mapping[str, str],
     support: _ResolvedSupport,
 ) -> tuple[str, ...]:
@@ -387,7 +395,7 @@ def _unsupported_params(
         ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
     )
     refused_image_param: Final = (
-        None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
+        None if provider_config.supports_images_in(model, ir_request.input, headers) else image_param(ir_request.input)
     )
     refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
     refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
@@ -414,6 +422,7 @@ def _is_not_box_question(question: DecisionsIRQuestion) -> bool:
 def _without_unsupported_params(
     ir_request: DecisionsIRRequest,
     provider_config: BaseDecisionsConfig,
+    model: str,
     headers: Mapping[str, str],
     support: _ResolvedSupport,
 ) -> DecisionsIRRequest:
@@ -421,7 +430,7 @@ def _without_unsupported_params(
         ir_request,
         input=(
             ir_request.input
-            if provider_config.supports_images_in(ir_request.input, headers)
+            if provider_config.supports_images_in(model, ir_request.input, headers)
             else with_images_where(ir_request.input, _no_image)
         ),
         safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
@@ -452,16 +461,17 @@ def _provider_ir_request(
     ir_request: DecisionsIRRequest,
     *,
     provider_config: BaseDecisionsConfig,
+    model: str,
     headers: Mapping[str, str],
     support: _ResolvedSupport,
     kwargs: Mapping[str, object],
 ) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
-    unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
+    unsupported: Final = _unsupported_params(ir_request, provider_config, model, headers, support)
     if not unsupported:
         return ir_request
     if not _drops_params(kwargs):
         return _UnsupportedParams(names=unsupported)
-    supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
+    supported: Final = _without_unsupported_params(ir_request, provider_config, model, headers, support)
     if _leaves_no_input(supported.input):
         return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
     if not supported.questions:
@@ -585,7 +595,12 @@ def prepare_call(
             llm_provider=provider,
         )
     ir_request: Final = _provider_ir_request(
-        caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
+        caller_ir_request,
+        provider_config=provider_config,
+        model=canonical_model,
+        headers=headers,
+        support=support,
+        kwargs=kwargs,
     )
     if isinstance(ir_request, _UnsupportedParams):
         raise litellm.UnsupportedParamsError(
@@ -643,9 +658,38 @@ def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]:
     return remote_image_urls(call.ir_request.input)
 
 
-def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall:
+def _with_images_replaced(call: DecisionsCall, replacements: Mapping[str, str]) -> DecisionsCall:
     return replace(
-        call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls))
+        call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, replacements))
+    )
+
+
+def _is_not_one_of(urls: frozenset[str], image: DecisionsImage) -> bool:
+    return not isinstance(image, str) or image not in urls
+
+
+def _leaves_nothing_to_decide_on(call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages) -> bool:
+    if _leaves_no_input(kept_input):
+        return True
+    return (
+        not call.provider_config.images_in_state
+        and not call.ir_request.audio
+        and not call.ir_request.videos
+        and not input_images(kept_input)
+        and input_has_no_text(kept_input)
+    )
+
+
+def _with_downloads_or_dropped_failures(
+    call: DecisionsCall, downloads: Mapping[str, str | litellm.ImageFetchError]
+) -> DecisionsCall:
+    failures: Final = tuple(result for result in downloads.values() if isinstance(result, litellm.ImageFetchError))
+    failed_urls: Final = frozenset(url for url, result in downloads.items() if not isinstance(result, str))
+    kept_input: Final = with_images_where(call.ir_request.input, partial(_is_not_one_of, failed_urls))
+    if failures and _leaves_nothing_to_decide_on(call, kept_input):
+        raise failures[0]
+    return _with_images_replaced(
+        call, MappingProxyType({url: result for url, result in downloads.items() if isinstance(result, str)})
     )
 
 
@@ -653,7 +697,12 @@ def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
     remote_urls: Final = _remote_image_urls(call)
     if not remote_urls:
         return call
-    return _with_data_urls(
+    if _drops_params(call.litellm_params):
+        return _with_downloads_or_dropped_failures(
+            call,
+            MappingProxyType({url: convert_url_to_base64_or_error(url, timeout=call.timeout) for url in remote_urls}),
+        )
+    return _with_images_replaced(
         call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls})
     )
 
@@ -662,8 +711,13 @@ async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCal
     remote_urls: Final = _remote_image_urls(call)
     if not remote_urls:
         return call
+    if _drops_params(call.litellm_params):
+        downloads: Final = await async_convert_urls_to_base64_or_errors(remote_urls, timeout=call.timeout)
+        return _with_downloads_or_dropped_failures(
+            call, MappingProxyType(dict(zip(remote_urls, downloads, strict=True)))
+        )
     data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
-    return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
+    return _with_images_replaced(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
 
 
 def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool:
@@ -671,57 +725,130 @@ def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bo
     return media_type is None or media_type in media_types
 
 
-def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
-    media_types: Final = call.provider_config.image_media_types
+def _within(limit: int | None, amount: int) -> bool:
+    return limit is None or amount <= limit
+
+
+def _media_type_error(media_types: frozenset[str] | None, images: Sequence[DecisionsImage]) -> str | None:
     if media_types is None:
-        return call
-    found: Final = (image_media_type(image) for image in input_images(call.ir_request.input))
+        return None
+    found: Final = (image_media_type(image) for image in images)
     unsupported: Final = tuple(
         dict.fromkeys(type_ for type_ in found if type_ is not None and type_ not in media_types)
     )
     if not unsupported:
-        return call
-    supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
-    if _drops_params(call.litellm_params) and not _leaves_no_input(supported):
-        return replace(call, ir_request=replace(call.ir_request, input=supported))
-    raise litellm.BadRequestError(
-        message=(
-            f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, "
-            f"got {', '.join(unsupported)}"
-        ),
+        return None
+    return f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}"
+
+
+def _count_error(max_images: int | None, image_count: int) -> str | None:
+    if _within(max_images, image_count):
+        return None
+    return f"takes at most {max_images} images per request, got {image_count}"
+
+
+def _refusal(call: DecisionsCall, refusal: str) -> litellm.BadRequestError:
+    return litellm.BadRequestError(
+        message=f"{call.custom_llm_provider} {refusal}",
         model=call.requested_model,
         llm_provider=call.custom_llm_provider,
     )
 
 
-def _checked_image_count(call: DecisionsCall) -> DecisionsCall:
-    max_images: Final = call.provider_config.max_images
-    image_count: Final = len(input_images(call.ir_request.input))
-    if max_images is None or image_count <= max_images:
+def _with_images_kept(
+    decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool]
+) -> DecisionsIRState | DecisionsIRMessages:
+    flags: Final = iter(kept)
+    return with_images_where(decision_input, lambda _image: next(flags))
+
+
+def _dropped_or_refused(
+    call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str
+) -> DecisionsCall:
+    if _drops_params(call.litellm_params) and not _leaves_nothing_to_decide_on(call, kept_input):
+        return replace(call, ir_request=replace(call.ir_request, input=kept_input))
+    raise _refusal(call, refusal)
+
+
+def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
+    media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types
+    error: Final = _media_type_error(media_types, input_images(call.ir_request.input))
+    if media_types is None or error is None:
         return call
-    raise litellm.BadRequestError(
-        message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}",
-        model=call.requested_model,
-        llm_provider=call.custom_llm_provider,
+    kept_input: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
+    if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input):
+        return replace(call, ir_request=replace(call.ir_request, input=kept_input))
+    raise _refusal(call, error)
+
+
+def _refused_over_count(call: DecisionsCall) -> DecisionsCall:
+    max_images: Final = call.provider_config.image_rules_for(call.model).max_images
+    error: Final = _count_error(max_images, len(input_images(call.ir_request.input)))
+    if error is None:
+        return call
+    past_the_count: Final = frozenset(_remote_image_urls(call)[max_images:])
+    return _dropped_or_refused(
+        call, with_images_where(call.ir_request.input, partial(_is_not_one_of, past_the_count)), error
     )
 
 
+def _decoded_byte_count(payload: str) -> int:
+    return len(standard_base64(payload).rstrip("=")) * 3 // 4
+
+
+def _base64_payload(image: DecisionsImage) -> str | None:
+    if isinstance(image, DecisionsBase64Image):
+        return image.base64
+    marker_at: Final = image.lower().find(_BASE64_MARKER)
+    return None if marker_at < 0 else image[marker_at + len(_BASE64_MARKER) :]
+
+
 def _decoded_image_bytes(image: DecisionsImage) -> int:
-    if isinstance(image, DecisionsBase64Image):
-        return len(image.base64.rstrip("=")) * 3 // 4
-    marker_at: Final = image.lower().find(_BASE64_MARKER)
-    return 0 if marker_at < 0 else len(image[marker_at + len(_BASE64_MARKER) :].rstrip("=")) * 3 // 4
+    payload: Final = _base64_payload(image)
+    return 0 if payload is None else _decoded_byte_count(payload)
+
+
+def _leading_bytes(payload: str) -> bytes:
+    head: Final = standard_base64(payload[:64])[:16]
+    try:
+        return base64.b64decode(head + "=" * (-len(head) % 4))
+    except ValueError:
+        return b""
+
+
+def _sniffed_media_types(head: bytes) -> frozenset[str]:
+    if head.startswith(b"\x89PNG\r\n\x1a\n"):
+        return frozenset({"image/png"})
+    if head.startswith(b"\xff\xd8\xff"):
+        return frozenset({"image/jpeg", "image/jpg"})
+    if head.startswith((b"GIF87a", b"GIF89a")):
+        return frozenset({"image/gif"})
+    if head.startswith(b"RIFF") and head[8:12] == b"WEBP":
+        return frozenset({"image/webp"})
+    return frozenset()
+
+
+def _is_readable_image(media_types: frozenset[str] | None, image: DecisionsImage) -> bool:
+    payload: Final = _base64_payload(image)
+    if media_types is None or not media_types <= _SNIFFABLE_MEDIA_TYPES or payload is None:
+        return True
+    return bool(_sniffed_media_types(_leading_bytes(payload)) & media_types)
+
+
+def _unreadable_image_error(call: DecisionsCall) -> str | None:
+    media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types
+    unreadable: Final = sum(not _is_readable_image(media_types, image) for image in input_images(call.ir_request.input))
+    if media_types is None or not unreadable:
+        return None
+    return f"takes only {', '.join(sorted(media_types))} image bytes, got {unreadable} image(s) that are none of these"
 
 
 def _byte_limit(limit: int) -> str:
     return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)"
 
 
-def _image_size_error(call: DecisionsCall) -> str | None:
-    sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
+def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None:
     largest: Final = max(sizes, default=0)
-    per_image: Final = call.provider_config.max_image_bytes
-    total: Final = call.provider_config.max_total_image_bytes
     if per_image is not None and largest > per_image:
         return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes"
     if total is not None and sum(sizes) > total:
@@ -729,17 +856,79 @@ def _image_size_error(call: DecisionsCall) -> str | None:
     return None
 
 
-def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall:
-    error: Final = _image_size_error(call)
-    if error is None:
-        return call
-    raise litellm.BadRequestError(
-        message=f"{call.custom_llm_provider} {error}",
-        model=call.requested_model,
-        llm_provider=call.custom_llm_provider,
+def _image_limit_error(call: DecisionsCall) -> str | None:
+    rules: Final = call.provider_config.image_rules_for(call.model)
+    images: Final = input_images(call.ir_request.input)
+    return (
+        _media_type_error(rules.image_media_types, images)
+        or _count_error(rules.max_images, len(images))
+        or _image_size_error(
+            tuple(_decoded_image_bytes(image) for image in images), rules.max_image_bytes, rules.max_total_image_bytes
+        )
     )
 
 
+def _fits_alone(rules: BaseDecisionsConfig, image: DecisionsImage) -> bool:
+    return (rules.image_media_types is None or _has_media_type_in(rules.image_media_types, image)) and _within(
+        rules.max_image_bytes, _decoded_image_bytes(image)
+    )
+
+
+def _is_usable(
+    rules: BaseDecisionsConfig, failed_downloads: frozenset[str], drops_params: bool, image: DecisionsImage
+) -> bool:
+    return (
+        _is_not_one_of(failed_downloads, image)
+        and _fits_alone(rules, image)
+        and (not drops_params or _is_readable_image(rules.image_media_types, image))
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _ImageBudget:
+    spent: int = 0
+    count: int = 0
+    kept: bool = False
+
+
+def _after_image(rules: BaseDecisionsConfig, budget: _ImageBudget, sized_fit: tuple[int, bool]) -> _ImageBudget:
+    size, fit = sized_fit
+    if not fit or not _within(rules.max_total_image_bytes, budget.spent + size):
+        return replace(budget, kept=False)
+    if not _within(rules.max_images, budget.count + 1):
+        return replace(budget, kept=False)
+    return _ImageBudget(spent=budget.spent + size, count=budget.count + 1, kept=True)
+
+
+def _kept_images(call: DecisionsCall) -> tuple[bool, ...]:
+    rules: Final = call.provider_config.image_rules_for(call.model)
+    images: Final = input_images(call.ir_request.input)
+    usable: Final = partial(_is_usable, rules, frozenset(_remote_image_urls(call)), _drops_params(call.litellm_params))
+    budgets: Final = accumulate(
+        zip(map(_decoded_image_bytes, images), map(usable, images), strict=True),
+        partial(_after_image, rules),
+        initial=_ImageBudget(),
+    )
+    return tuple(budget.kept for budget in islice(budgets, 1, None))
+
+
+def _drops_images(before: DecisionsCall, after: DecisionsCall) -> bool:
+    return len(input_images(after.ir_request.input)) < len(input_images(before.ir_request.input))
+
+
+def _with_images_within_limits(call: DecisionsCall, kept: Sequence[bool], dropped_past_count: bool) -> DecisionsCall:
+    if all(kept):
+        return call
+    kept_input: Final = _with_images_kept(call.ir_request.input, kept)
+    error: Final = _image_limit_error(call) or _unreadable_image_error(call)
+    if error is None:
+        return replace(call, ir_request=replace(call.ir_request, input=kept_input))
+    type_checked: Final = _checked_image_types(call)
+    if not dropped_past_count and input_images(type_checked.ir_request.input) == input_images(kept_input):
+        return type_checked
+    return _dropped_or_refused(call, kept_input, error)
+
+
 def _request_body(call: DecisionsCall) -> Mapping[str, object]:
     body: Final = call.provider_config.transform_decisions_request(
         model=call.model, request=call.ir_request, custom_llm_provider=call.custom_llm_provider
@@ -753,6 +942,19 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
     return body
 
 
+def _logged_body(
+    supported: DecisionsCall, fetched: DecisionsCall, kept: Sequence[bool], body: Mapping[str, object]
+) -> Mapping[str, object]:
+    if fetched is supported:
+        return body
+    return _request_body(
+        replace(
+            supported,
+            ir_request=replace(supported.ir_request, input=_with_images_kept(supported.ir_request.input, kept)),
+        )
+    )
+
+
 def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
     returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
     if returned is not None and cost_map_key_prices_anything(returned):
@@ -816,24 +1018,105 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
             )
 
 
+_RESENT_WITHOUT_IMAGES_STATUSES: Final = frozenset({400, 408, 413, 422})
+
+
+def _may_be_caused_by_images(error: BaseLLMException | litellm.Timeout) -> bool:
+    if isinstance(error, litellm.Timeout):
+        return True
+    return not error.status_code_is_synthesized and error.status_code in _RESENT_WITHOUT_IMAGES_STATUSES
+
+
+def _goes_again_without_images(call: DecisionsCall, error: BaseLLMException | litellm.Timeout) -> bool:
+    return (
+        call.provider_config.images_in_state and _drops_params(call.litellm_params) and _may_be_caused_by_images(error)
+    )
+
+
+def _without_images_after_refusal(
+    call: DecisionsCall, error: BaseLLMException | litellm.Timeout
+) -> DecisionsCall | None:
+    images: Final = input_images(call.ir_request.input)
+    if not images or not _goes_again_without_images(call, error):
+        return None
+    without_images: Final = _with_images_kept(call.ir_request.input, tuple(False for _ in images))
+    if _leaves_no_input(without_images):
+        return None
+    return replace(call, ir_request=replace(call.ir_request, input=without_images))
+
+
+async def _aposted(
+    call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object]
+) -> DecisionsIRResponse:
+    return await _HANDLER.adecisions(
+        model=call.model,
+        custom_llm_provider=call.custom_llm_provider,
+        logging_obj=call.logging_obj,
+        provider_config=call.provider_config,
+        request=call.ir_request,
+        body=body,
+        logged_body=logged_body,
+        api_base=call.api_base,
+        api_key=call.api_key,
+        headers=call.headers,
+        timeout=call.timeout,
+        litellm_params=call.litellm_params,
+    )
+
+
+def _posted(call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object]) -> DecisionsIRResponse:
+    return _HANDLER.decisions(
+        model=call.model,
+        custom_llm_provider=call.custom_llm_provider,
+        logging_obj=call.logging_obj,
+        provider_config=call.provider_config,
+        request=call.ir_request,
+        body=body,
+        logged_body=logged_body,
+        api_base=call.api_base,
+        api_key=call.api_key,
+        headers=call.headers,
+        timeout=call.timeout,
+        litellm_params=call.litellm_params,
+    )
+
+
+async def _aposted_without_refused_images(
+    call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object]
+) -> DecisionsIRResponse:
+    try:
+        return await _aposted(call, body, logged_body)
+    except (BaseLLMException, litellm.Timeout) as error:
+        retry: Final = _without_images_after_refusal(call, error)
+        if retry is None:
+            raise
+        retry_body: Final = _request_body(retry)
+        return await _aposted(retry, retry_body, retry_body)
+
+
+def _posted_without_refused_images(
+    call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object]
+) -> DecisionsIRResponse:
+    try:
+        return _posted(call, body, logged_body)
+    except (BaseLLMException, litellm.Timeout) as error:
+        retry: Final = _without_images_after_refusal(call, error)
+        if retry is None:
+            raise
+        retry_body: Final = _request_body(retry)
+        return _posted(retry, retry_body, retry_body)
+
+
 async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
-    supported: Final = _checked_image_count(_checked_image_types(call))
-    inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported)))
+    type_checked: Final = _checked_image_types(call)
+    supported: Final = _refused_over_count(type_checked)
+    fetched: Final = await _async_with_remote_images_inlined(supported)
+    kept: Final = _kept_images(fetched)
+    inlined: Final = _with_images_within_limits(fetched, kept, _drops_images(type_checked, supported))
     body: Final = _request_body(inlined)
     try:
-        response: Final = await _HANDLER.adecisions(
-            model=inlined.model,
-            custom_llm_provider=inlined.custom_llm_provider,
-            logging_obj=inlined.logging_obj,
-            provider_config=inlined.provider_config,
-            request=inlined.ir_request,
-            body=body,
-            logged_body=body if inlined is supported else _request_body(supported),
-            api_base=inlined.api_base,
-            api_key=inlined.api_key,
-            headers=inlined.headers,
-            timeout=inlined.timeout,
-            litellm_params=inlined.litellm_params,
+        response: Final = await _aposted_without_refused_images(
+            inlined, body, _logged_body(supported, fetched, kept, body)
         )
     except Exception as error:
         raise _map_upstream_exception(error, inlined) from error
@@ -841,24 +1124,14 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon
 
 
 def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
-    supported: Final = _checked_image_count(_checked_image_types(call))
-    inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported)))
+    type_checked: Final = _checked_image_types(call)
+    supported: Final = _refused_over_count(type_checked)
+    fetched: Final = _with_remote_images_inlined(supported)
+    kept: Final = _kept_images(fetched)
+    inlined: Final = _with_images_within_limits(fetched, kept, _drops_images(type_checked, supported))
     body: Final = _request_body(inlined)
     try:
-        response: Final = _HANDLER.decisions(
-            model=inlined.model,
-            custom_llm_provider=inlined.custom_llm_provider,
-            logging_obj=inlined.logging_obj,
-            provider_config=inlined.provider_config,
-            request=inlined.ir_request,
-            body=body,
-            logged_body=body if inlined is supported else _request_body(supported),
-            api_base=inlined.api_base,
-            api_key=inlined.api_key,
-            headers=inlined.headers,
-            timeout=inlined.timeout,
-            litellm_params=inlined.litellm_params,
-        )
+        response: Final = _posted_without_refused_images(inlined, body, _logged_body(supported, fetched, kept, body))
     except Exception as error:
         raise _map_upstream_exception(error, inlined) from error
     return _format_response(response, inlined)

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -522,6 +522,31 @@ async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -
         raise
 
 
+def convert_url_to_base64_or_error(
+    url: str, timeout: float | httpx.Timeout | None = None
+) -> "str | litellm.ImageFetchError":
+    try:
+        return convert_url_to_base64(url, timeout=timeout)
+    except litellm.ImageFetchError as error:
+        return error
+
+
+async def _fetch_data_url_or_error(
+    url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None
+) -> "str | litellm.ImageFetchError":
+    try:
+        return await _fetch_data_url(url, in_flight, timeout)
+    except litellm.ImageFetchError as error:
+        return error
+
+
+async def async_convert_urls_to_base64_or_errors(
+    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
+) -> "tuple[str | litellm.ImageFetchError, ...]":
+    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
+    return tuple(await asyncio.gather(*(_fetch_data_url_or_error(url, in_flight, timeout) for url in remote_urls)))
+
+
 async def async_convert_urls_to_base64(
     remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
 ) -> tuple[str, ...]:

litellm/llms/base_llm/decisions/transformation.py

@@ -4,14 +4,15 @@ from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, replace
 from itertools import chain
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeAlias
 
 import httpx
 from pydantic import TypeAdapter, ValidationError
 from pydantic_core import ErrorDetails
-from typing_extensions import assert_never
+from typing_extensions import TypeIs, assert_never  # noqa: TID251  # TypeIs reaches typing only on 3.13
 
 import litellm
+from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -66,12 +67,20 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter
 _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
 _REMOTE_URL_PREFIXES: Final = ("http://", "https://")
 _DATA_URL_PREFIX: Final = "data:"
+_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/")
+_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"})
+
+_StateItem: TypeAlias = str | Mapping[str, object]
 
 
 def decisions_text(value: DecisionsJSON) -> str:
     return value if isinstance(value, str) else json.dumps(value)
 
 
+def standard_base64(value: str) -> str:
+    return "".join(value.split()).translate(_URL_SAFE_TO_STANDARD_BASE64)
+
+
 def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]:
     names: Final = tuple(question.name for question in questions if question.name is not None)
     if len(frozenset(names)) == len(questions):
@@ -240,6 +249,34 @@ def _systemone_state(decision_input: DecisionsIRState | DecisionsIRMessages) ->
             assert_never(decision_input)
 
 
+def _is_json_object(value: object) -> TypeIs[Mapping[str, object]]:  # guard-ok: JSON object keys are strings
+    return isinstance(value, Mapping)
+
+
+def _is_json_array(value: object) -> TypeIs[Sequence[object]]:  # guard-ok: only the items are read
+    return isinstance(value, Sequence)
+
+
+def _holds_no_text(value: object, depth: int = 0) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, str):
+        return not value.strip()
+    if depth >= DEFAULT_MAX_RECURSE_DEPTH:
+        return True
+    if _is_json_object(value):
+        return all(
+            _holds_no_text(item, depth + 1) for key, item in value.items() if key != "type" or "text" not in value
+        )
+    if _is_json_array(value):
+        return all(_holds_no_text(item, depth + 1) for item in value)
+    return False
+
+
+def input_has_no_text(decision_input: DecisionsIRState | DecisionsIRMessages) -> bool:
+    return _holds_no_text(_systemone_state(decision_input))
+
+
 def _media_type(content_type: str) -> str:
     return content_type.split(";", 1)[0].strip().lower()
 
@@ -261,6 +298,54 @@ def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]:
     return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}"
 
 
+def _standard_data_url(media_type: str, payload: str) -> str:
+    standard_payload: Final = standard_base64(payload)
+    padding: Final = "=" * (-len(standard_payload) % 4)
+    standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type)
+    return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_payload}{padding}"
+
+
+def _data_url(image: DecisionsImage) -> str:
+    if isinstance(image, DecisionsBase64Image):
+        return _standard_data_url(_media_type(image.content_type), image.base64)
+    media_type: Final = image_media_type(image)
+    if media_type is None:
+        return image
+    return _standard_data_url(media_type, image.partition(",")[2])
+
+
+def _image_item(image: DecisionsImage) -> _StateItem:
+    return {"type": "image_url", "image_url": {"url": _data_url(image)}}
+
+
+def _part_item(part: OpenAIDecisionInputText | OpenAIDecisionInputImage) -> _StateItem:
+    if isinstance(part, OpenAIDecisionInputText):
+        return part.text
+    return _image_item(part.image_url)
+
+
+def _message_items(message: OpenAIDecisionInputMessage) -> tuple[_StateItem, ...]:
+    if isinstance(message.content, str):
+        return (message.content,)
+    return tuple(map(_part_item, message.content))
+
+
+def _state_items(state: DecisionsJSON) -> tuple[object, ...]:
+    if isinstance(state, (str, Mapping)):
+        return (state,)
+    return tuple(state)
+
+
+def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[object, ...]:
+    match decision_input:
+        case DecisionsIRState():
+            return (*_state_items(decision_input.state), *map(_image_item, decision_input.images))
+        case DecisionsIRMessages():
+            return tuple(chain.from_iterable(map(_message_items, decision_input.messages)))
+        case _:
+            assert_never(decision_input)
+
+
 def _optional(key: str, value: object) -> Mapping[str, object]:
     return {} if value is None else {key: value}
 
@@ -298,13 +383,15 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
             assert_never(question)
 
 
-def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
-    state: Final = _systemone_state(request.input)
+def ir_to_systemone_request(
+    model: str, request: DecisionsIRRequest, *, images_in_state: bool = False
+) -> Mapping[str, object]:
     keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
     images: Final = input_images(request.input)
+    state_carries_images: Final = images_in_state and bool(images)
     return {
         "model": model,
-        "state": state,
+        "state": [*_multimodal_state(request.input)] if state_carries_images else _systemone_state(request.input),
         **(
             {"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]}
             if request.audio
@@ -316,7 +403,7 @@ def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[
             else {}
         ),
         "questions": {key: _systemone_question(question) for key, question in keyed_questions},
-        **({"images": [_systemone_image(image) for image in images]} if images else {}),
+        **({"images": [_systemone_image(image) for image in images]} if images and not state_carries_images else {}),
         **(
             {"assets": {key: asset.model_dump(exclude_none=True) for key, asset in request.assets.items()}}
             if request.assets
@@ -538,6 +625,7 @@ class BaseDecisionsConfig(ABC):
     max_total_image_bytes: int | None = None
     image_media_types: frozenset[str] | None = None
     inlines_remote_images: bool = False
+    images_in_state: bool = False
     openai_evidence_fields: frozenset[str] = frozenset()
     confidence_optional: bool = False
     health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
@@ -635,10 +723,13 @@ class BaseDecisionsConfig(ABC):
         return headers, None
 
     def supports_images_in(
-        self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
+        self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
     ) -> bool:
         return self.supports_images
 
+    def image_rules_for(self, model: str) -> "BaseDecisionsConfig":
+        return self
+
     def supports_assets(self, headers: Mapping[str, str]) -> bool:
         return False
 
@@ -671,7 +762,7 @@ class BaseDecisionsConfig(ABC):
         request: DecisionsIRRequest,
         custom_llm_provider: str,
     ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
-        return ir_to_systemone_request(self.request_model(model), request)
+        return ir_to_systemone_request(self.request_model(model), request, images_in_state=self.images_in_state)
 
     def unwrap_response(self, payload: object) -> object:
         return payload

litellm/llms/openrouter/decisions/transformation.py

@@ -1,17 +1,44 @@
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
-from litellm.types.decisions import DecisionsIRResponse
+from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig
+from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig
+from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig
+from litellm.types.decisions import DecisionsIRMessages, DecisionsIRResponse, DecisionsIRState
+
+_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType(
+    {
+        "typesafe": TypeSafeDecisionsConfig(),
+        "cloudflare": CloudflareDecisionsConfig(),
+        "perplexity": PerplexityDecisionsConfig(),
+    }
+)
+
+
+def _upstream_org(model: str) -> str:
+    return model.removeprefix("~").partition("/")[0]
 
 
 class OpenRouterDecisionsConfig(BaseDecisionsConfig):
     path = "/alpha/decisions"
     api_key_env = ("OPENROUTER_API_KEY",)
     api_base_env = ("OPENROUTER_API_BASE",)
+    inlines_remote_images = True
+    images_in_state = True
 
     def get_default_api_base(self) -> str | None:
         return "https://openrouter.ai/api"
 
+    def supports_images_in(
+        self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
+    ) -> bool:
+        return _upstream_org(model) in _ORG_IMAGE_RULES
+
+    def image_rules_for(self, model: str) -> BaseDecisionsConfig:
+        return _ORG_IMAGE_RULES.get(_upstream_org(model), self)
+
     def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
         cost: Final = response.usage.extra.get("cost") if response.usage is not None else None
         if not isinstance(cost, (int, float)):

litellm/llms/perplexity/decisions/transformation.py

@@ -5,6 +5,11 @@ class PerplexityDecisionsConfig(BaseDecisionsConfig):
     path = "/v1/decisions"
     api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY")
     api_base_env = ("PERPLEXITY_API_BASE",)
+    supports_images = True
+    max_images = 8
+    image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"})
+    inlines_remote_images = True
+    images_in_state = True
 
     def get_default_api_base(self) -> str | None:
         return "https://api.perplexity.ai"

litellm/llms/typesafe/decisions/transformation.py

@@ -6,7 +6,12 @@ from types import MappingProxyType
 from typing import Final, Literal
 
 from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX
-from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, DecisionsInputLimits, image_media_type
+from litellm.llms.base_llm.decisions.transformation import (
+    BaseDecisionsConfig,
+    DecisionsInputLimits,
+    image_media_type,
+    standard_base64,
+)
 from litellm.types.decisions import (
     DecisionsAsset,
     DecisionsIRMessages,
@@ -23,7 +28,6 @@ _LONG_CONTEXT_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}long-context
 _ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType(
     {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"}
 )
-_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/")
 
 
 def _image_reference(image_url: str) -> str:
@@ -37,7 +41,7 @@ def _image_asset(image_url: str) -> DecisionsAsset | None:
     return DecisionsAsset(
         type="image",
         format=asset_format,
-        data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64),
+        data_base64=standard_base64(image_url.partition(",")[2]),
     )
 
 
@@ -96,7 +100,7 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig):
         return DecisionsInputLimits(max_request_tokens=65536, max_question_tokens=32768)
 
     def supports_images_in(
-        self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
+        self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
     ) -> bool:
         return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers)
 

litellm/model_prices_and_context_window_backup.json

@@ -47256,6 +47256,18 @@
         "supports_system_messages": true,
         "supports_vision": false
     },
+    "openrouter/typesafe/jev-1.14": {
+        "litellm_provider": "openrouter",
+        "mode": "evaluation",
+        "source": "https://openrouter.ai/docs/guides/community/jev",
+        "supports_box_questions": true
+    },
+    "openrouter/~typesafe/jev-latest": {
+        "litellm_provider": "openrouter",
+        "mode": "evaluation",
+        "source": "https://openrouter.ai/docs/guides/community/jev",
+        "supports_box_questions": true
+    },
     "ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
         "input_cost_per_token": 6.7e-07,
         "litellm_provider": "ovhcloud",

model_prices_and_context_window.json

@@ -47256,6 +47256,18 @@
         "supports_system_messages": true,
         "supports_vision": false
     },
+    "openrouter/typesafe/jev-1.14": {
+        "litellm_provider": "openrouter",
+        "mode": "evaluation",
+        "source": "https://openrouter.ai/docs/guides/community/jev",
+        "supports_box_questions": true
+    },
+    "openrouter/~typesafe/jev-latest": {
+        "litellm_provider": "openrouter",
+        "mode": "evaluation",
+        "source": "https://openrouter.ai/docs/guides/community/jev",
+        "supports_box_questions": true
+    },
     "ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
         "input_cost_per_token": 6.7e-07,
         "litellm_provider": "ovhcloud",

tests/code_coverage_tests/recursive_detector.py

@@ -51,6 +51,7 @@ IGNORE_FUNCTIONS = [
     "_convert_to_json_serializable_dict",  # max depth set (default 20) and circular reference protection to prevent infinite recursion.
     "dict",  # max depth set. _LiteLLMParamsDictView.dict() calls builtin dict(), not itself.
     "_read_image_bytes",  # max depth set.
+    "_holds_no_text",  # max depth set.
     "get_masked_values",  # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts.
     "_redact_sensitive_litellm_params",  # max depth set (default 10).
     "_redact_secret_values_in_obj",  # max depth set (default 10, _REDACT_SECRET_MAX_DEPTH); fails closed by returning "REDACTED" at the cap.

tests/integration/providers/test_cloudflare_clef_images_wire.py

@@ -321,7 +321,7 @@ def test_an_openai_decision_model_gets_system_one_images_as_input_image_parts(ga
 
 
 @pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions"))
-def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream_call(
+def test_more_images_than_clef_takes_are_refused_before_any_download_or_trimmed_under_drop_params(
     gateway: Gateway, route: Route
 ) -> None:
     marker: Final = uuid.uuid4().hex
@@ -329,12 +329,15 @@ def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream
         strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
         dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
         remote: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(5)]
-        for model in (strict, dropping):
-            refused = _decide(gateway, route, model, marker, remote)
-            assert refused.status_code == 400, refused.text
-            assert "at most 4 images per request, got 5" in refused.text, refused.text
+        refused: Final = _decide(gateway, route, strict, marker, remote)
+        assert refused.status_code == 400, refused.text
+        assert "at most 4 images per request, got 5" in refused.text, refused.text
         assert wire.drain() == ()
         four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL]
+        trimmed: Final = _decide(gateway, route, dropping, marker, [*four, _PNG_DATA_URL])
+        assert trimmed.status_code == 200, trimmed.text
+        (dropped_request,) = wire.drain()
+        assert _upstream_body(dropped_request) == _expected_upstream_body("clef", _state(marker), four)
         accepted: Final = _decide(gateway, route, strict, marker, four)
         assert accepted.status_code == 200, accepted.text
         (request,) = wire.drain()
@@ -500,41 +503,55 @@ def test_an_image_clef_refuses_does_not_count_toward_its_limit_when_the_deployme
 _MIB: Final = 1024 * 1024
 
 
+_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n"
+
+
+def _png_base64(decoded_bytes: int) -> str:
+    return base64.b64encode(_PNG_SIGNATURE.ljust(decoded_bytes, b"\0")).decode()
+
+
 def _png_data_url(decoded_bytes: int) -> str:
-    return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}"
+    return f"data:image/png;base64,{_png_base64(decoded_bytes)}"
 
 
 def _png_object(decoded_bytes: int) -> dict[str, JsonValue]:
-    return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()}
+    return {"content_type": "image/png", "base64": _png_base64(decoded_bytes)}
 
 
-_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], str], ...]] = (
-    ("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1),), "got one of 4,194,305 bytes"),
-    ("one_over_4mib", "/v1/decisions", (_png_data_url(4 * _MIB + 1),), "got one of 4,194,305 bytes"),
+_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], int, str], ...]] = (
+    ("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1), _PNG_DATA_URL), 0, "got one of 4,194,305 bytes"),
+    ("one_over_4mib", "/v1/decisions", (_PNG_DATA_URL, _png_data_url(4 * _MIB + 1)), 1, "got one of 4,194,305 bytes"),
     (
         "total_over_8mib",
         "/v1/systemone",
-        (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)),
-        "got 8,388,609 bytes",
+        (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(8)),
+        2,
+        "got 8,388,616 bytes",
     ),
 )
 
 
 @pytest.mark.parametrize(
-    ("label", "route", "images", "refusal"), _OVERSIZED, ids=[f"{label}{route}" for label, route, _, _ in _OVERSIZED]
+    ("label", "route", "images", "first_dropped", "refusal"),
+    _OVERSIZED,
+    ids=[f"{label}{route}" for label, route, _, _, _ in _OVERSIZED],
 )
-def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call(
-    gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], refusal: str
+def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call_or_dropped_under_drop_params(
+    gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], first_dropped: int, refusal: str
 ) -> None:
     marker: Final = uuid.uuid4().hex
     with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
         strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
         dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
-        for model in (strict, dropping):
-            refused = _decide(gateway, route, model, marker, images)
-            assert refused.status_code == 400, (label, refused.text[:500])
-            assert refusal in refused.text, (label, refused.text[:500])
+        refused: Final = _decide(gateway, route, strict, marker, images)
+        assert refused.status_code == 400, (label, refused.text[:500])
+        assert refusal in refused.text, (label, refused.text[:500])
         assert wire.drain() == (), label
+        kept: Final = (*images[:first_dropped], *images[first_dropped + 1 :])
+        trimmed: Final = _decide(gateway, route, dropping, marker, images)
+        assert trimmed.status_code == 200, (label, trimmed.text[:500])
+        (dropped_request,) = wire.drain()
+        assert _upstream_body(dropped_request) == _expected_upstream_body("clef", _state(marker), kept), label
         at_limits: Final[list[JsonValue]] = [_png_data_url(4 * _MIB), _png_object(4 * _MIB)]
         accepted: Final = _decide(gateway, "/v1/systemone", strict, marker, at_limits)
         assert accepted.status_code == 200, accepted.text[:500]
@@ -848,6 +865,16 @@ def test_a_missing_remote_image_is_refused_without_a_call_to_clef(allowlisted: G
 
 
 @pytest.mark.timeout(_OWNED_TIMEOUT)
+def test_remote_images_past_clefs_count_are_never_downloaded_under_drop_params(
+    allowlisted: Gateway, wire: Wire
+) -> None:
+    marker: Final = uuid.uuid4().hex
+    images: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(6)]
+    response: Final = _decide(allowlisted, "/v1/systemone", _ALLOWLISTED_MODEL, marker, images, drop_params=True)
+    assert response.status_code == 200, response.text
+    assert sorted(method for method, _ in _routes(wire.drain())) == ["GET"] * 4 + ["POST"]
+
+
 def test_a_remote_page_that_is_not_an_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None:
     marker: Final = uuid.uuid4().hex
     response: Final = _decide(

tests/integration/providers/test_decisions_openai_format_wire.py

@@ -162,6 +162,7 @@ _PROVIDERS: Final = (
         _API_KEY,
         False,
         "perplexity/pplx-decider-v1-27b",
+        takes_images=True,
     ),
     _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"),
     _Provider(
@@ -172,6 +173,7 @@ _PROVIDERS: Final = (
         _API_KEY,
         False,
         None,
+        takes_images=True,
         provider_reported_cost=1.5834e-5,
     ),
     _Provider(
@@ -224,9 +226,10 @@ _PROVIDERS: Final = (
 _SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai)
 _TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images)
 _PERPLEXITY: Final = _PROVIDERS[0]
+_OPENROUTER: Final = next(provider for provider in _PROVIDERS if provider.name == "openrouter")
 _TYPESAFE: Final = _PROVIDERS[1]
 _OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai)
-_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images)
+_CLOUDFLARE: Final = next(provider for provider in _PROVIDERS if provider.name == "cloudflare")
 _PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"}
 _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
     ("missing questions", {"input": _INPUT}),
@@ -519,6 +522,21 @@ def test_cloudflare_receives_images_from_either_format_in_its_images_field(gatew
         assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2
 
 
+@pytest.mark.parametrize("provider", (_PERPLEXITY, _OPENROUTER), ids=_provider_id)
+def test_perplexity_and_openrouter_receive_images_from_either_format_in_their_state(
+    gateway: Gateway, provider: _Provider
+) -> None:
+    with gateway.scenario() as scenario:
+        handle: Final = _register(scenario, provider.upstream_reply())
+        model: Final = _deployment(scenario, handle, provider)
+        openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model)
+        assert openai_format.status_code == 200, openai_format.text
+        assert system_one_format.status_code == 200, system_one_format.text
+        image_item: Final[dict[str, JsonValue]] = {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}
+        expected_body: Final = {**provider.upstream_body(), "state": [_INPUT, image_item]}
+        assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2
+
+
 def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None:
     with gateway.scenario() as scenario:
         handle: Final = _register(scenario, _OPENAI.upstream_reply())

tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py

@@ -1,5 +1,7 @@
 import json
+import sys
 from collections.abc import Mapping
+from functools import reduce
 from typing import Final
 
 import httpx
@@ -13,6 +15,7 @@ from litellm.llms.base_llm.decisions.transformation import (
     BaseDecisionsConfig,
     DecisionsInputLimits,
     DecisionsResponseValidationError,
+    input_has_no_text,
     ir_to_systemone_request,
     ir_to_systemone_response,
     parse_systemone_response,
@@ -26,6 +29,7 @@ from litellm.types.decisions import (
     DecisionsIRRefusal,
     DecisionsIRRequest,
     DecisionsIRScoreAnswer,
+    DecisionsIRState,
     DecisionsRequestBody,
     OpenAIDecisionRequestBody,
 )
@@ -517,3 +521,13 @@ def test_non_json_provider_response_is_an_api_error(raw_content: bytes, provider
     with pytest.raises(BaseLLMException, match="unexpected response") as error:
         config.transform_decisions_response("drex-v2.0", provider, httpx.Response(200, content=raw_content), request)
     assert error.value.status_code == 500
+
+
+def _nested_once(inner: object, _: int) -> object:
+    return [inner]
+
+
+def test_input_has_no_text_counts_a_state_nested_past_the_stack_as_no_text() -> None:
+    deepest_state: Final = reduce(_nested_once, range(sys.getrecursionlimit() * 2), "Product photo")
+
+    assert input_has_no_text(DecisionsIRState(state=deepest_state, images=())) is True

tests/unit/proxy/decisions_endpoints/test_endpoints.py

@@ -937,7 +937,7 @@ def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_p
 
 
 _IMAGE_TEXT: Final = "Is the screen cracked?"
-_PNG_DATA_URL: Final = "data:image/png;base64,AA=="
+_PNG_DATA_URL: Final = "data:image/png;base64,iVBORw0KGgo="
 _IMAGE_ENDPOINTS: Final = ("/v1/decisions", "/v1/systemone")
 _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = (
     {
@@ -962,6 +962,8 @@ _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = (
 )
 _IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True))
 _IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format")
+_TEXT_ONLY_DEPLOYMENT: Final[Mapping[str, object]] = {"model": "nace/drex-v2.0", "api_key": "test-key"}
+_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone"
 
 
 def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None:
@@ -1017,7 +1019,8 @@ def test_images_to_a_text_only_deployment_are_refused_without_drop_params(
     param: str,
 ) -> None:
     monkeypatch.setattr(litellm, "drop_params", False)
-    upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
+    _route_decider_to(monkeypatch, _TEXT_ONLY_DEPLOYMENT)
+    upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
 
     response: Final = client.post(endpoint, json=request_body)
 
@@ -1035,16 +1038,14 @@ def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params(
     request_body: Mapping[str, object],
 ) -> None:
     monkeypatch.setattr(litellm, "drop_params", False)
-    _route_decider_to(
-        monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True}
-    )
-    upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
+    _route_decider_to(monkeypatch, {**_TEXT_ONLY_DEPLOYMENT, "drop_params": True})
+    upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
 
     response: Final = client.post(endpoint, json=request_body)
 
     assert response.status_code == 200, response.text
     assert json.loads(upstream.calls[0].request.content) == {
-        "model": "pplx-decider-v1-27b",
+        "model": "drex-v2.0",
         "state": _IMAGE_TEXT,
         "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
     }
@@ -1104,7 +1105,12 @@ def test_malformed_assets_for_a_deployment_without_asset_support_are_dropped_onl
 
     response: Final = client.post(
         "/v1/systemone",
-        json={**_IMAGE_REQUEST_BODIES[1], "assets": {"photo-1": {"type": "image", "part": "file1"}}},
+        json={
+            "model": "decider",
+            "state": _IMAGE_TEXT,
+            "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
+            "assets": {"photo-1": {"type": "image", "part": "file1"}},
+        },
     )
 
     if not deployment_drops_params: