From 08c5a183a8cbbd4c30acc000956b862fdac76873 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:57:49 -0700 Subject: [PATCH] fix(decisions): follow each OpenRouter upstream's image rules, keep caller state items, normalize URL-safe base64 OpenRouter decision models now take the image types, count and size limits of the provider behind them (TypeSafe or Cloudflare), replacing the TypeSafe-only media type override with one rules lookup. Perplexity caps images at 8 before the call. A JSON object state stays one object item and a list state keeps its items ahead of the images, and URL-safe base64 in data URLs is converted to standard base64, which every upstream requires. --- litellm/decisions/call.py | 9 ++- .../llms/base_llm/decisions/transformation.py | 23 ++++-- .../openrouter/decisions/transformation.py | 14 ++-- .../perplexity/decisions/transformation.py | 1 + .../llms/typesafe/decisions/transformation.py | 5 +- tests/unit/decisions/test_main.py | 80 ++++++++++++++++++- 6 files changed, 110 insertions(+), 22 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index ca0acc0aa95..63bd1222d3f 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -612,7 +612,7 @@ def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bo def _checked_image_types(call: DecisionsCall) -> DecisionsCall: - media_types: Final = call.provider_config.image_media_types_for(call.model) + media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types if media_types is None: return call found: Final = (image_media_type(image) for image in input_images(call.ir_request.input)) @@ -635,7 +635,7 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall: def _checked_image_count(call: DecisionsCall) -> DecisionsCall: - max_images: Final = call.provider_config.max_images + max_images: Final = call.provider_config.image_rules_for(call.model).max_images image_count: Final = len(input_images(call.ir_request.input)) if max_images is None or image_count <= max_images: return call @@ -660,8 +660,9 @@ def _byte_limit(limit: int) -> str: def _image_size_error(call: DecisionsCall) -> str | None: sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) largest: Final = max(sizes, default=0) - per_image: Final = call.provider_config.max_image_bytes - total: Final = call.provider_config.max_total_image_bytes + rules: Final = call.provider_config.image_rules_for(call.model) + per_image: Final = rules.max_image_bytes + total: Final = rules.max_total_image_bytes if per_image is not None and largest > per_image: return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes" if total is not None and sum(sizes) > total: diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index b6115160ac5..50dacadbe07 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -65,6 +65,7 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" +_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") _StateItem: TypeAlias = str | Mapping[str, object] @@ -73,6 +74,10 @@ def decisions_text(value: DecisionsJSON) -> str: return value if isinstance(value, str) else json.dumps(value) +def standard_base64(value: str) -> str: + return value.translate(_URL_SAFE_TO_STANDARD_BASE64) + + def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]: names: Final = tuple(question.name for question in questions if question.name is not None) if len(frozenset(names)) == len(questions): @@ -262,10 +267,10 @@ def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]: def _data_url(image: DecisionsImage) -> str: media_type: Final = image_media_type(image) if isinstance(image, DecisionsBase64Image): - return f"{_DATA_URL_PREFIX}{media_type};base64,{image.base64}" + return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.base64)}" if media_type is None: return image - return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}" + return f"{_DATA_URL_PREFIX}{media_type};base64,{standard_base64(image.partition(',')[2])}" def _image_item(image: DecisionsImage) -> _StateItem: @@ -284,10 +289,16 @@ def _message_items(message: OpenAIDecisionInputMessage) -> tuple[_StateItem, ... return tuple(map(_part_item, message.content)) -def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[_StateItem, ...]: +def _state_items(state: DecisionsJSON) -> tuple[object, ...]: + if isinstance(state, (str, Mapping)): + return (state,) + return tuple(state) + + +def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[object, ...]: match decision_input: case DecisionsIRState(): - return (decisions_text(decision_input.state), *map(_image_item, decision_input.images)) + return (*_state_items(decision_input.state), *map(_image_item, decision_input.images)) case DecisionsIRMessages(): return tuple(chain.from_iterable(map(_message_items, decision_input.messages))) case _: @@ -649,8 +660,8 @@ class BaseDecisionsConfig(ABC): ) -> bool: return self.supports_images - def image_media_types_for(self, model: str) -> frozenset[str] | None: - return self.image_media_types + def image_rules_for(self, model: str) -> "BaseDecisionsConfig": + return self def supports_assets(self, headers: Mapping[str, str]) -> bool: return False diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index ef336f7c336..b1af997920b 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,9 +1,15 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig +from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig from litellm.types.decisions import DecisionsIRResponse -_TYPESAFE_IMAGE_MEDIA_TYPES: Final = frozenset({"image/png", "image/jpeg", "image/jpg"}) +_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( + {"typesafe": TypeSafeDecisionsConfig(), "cloudflare": CloudflareDecisionsConfig()} +) class OpenRouterDecisionsConfig(BaseDecisionsConfig): @@ -18,10 +24,8 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): def get_default_api_base(self) -> str | None: return "https://openrouter.ai/api" - def image_media_types_for(self, model: str) -> frozenset[str] | None: - if model.removeprefix("~").startswith("typesafe/"): - return _TYPESAFE_IMAGE_MEDIA_TYPES - return self.image_media_types + def image_rules_for(self, model: str) -> BaseDecisionsConfig: + return _ORG_IMAGE_RULES.get(model.removeprefix("~").partition("/")[0], self) def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: cost: Final = response.usage.extra.get("cost") if response.usage is not None else None diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 6ffc0cfdb41..06989ba4134 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -6,6 +6,7 @@ class PerplexityDecisionsConfig(BaseDecisionsConfig): api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") api_base_env = ("PERPLEXITY_API_BASE",) supports_images = True + max_images = 8 image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) inlines_remote_images = True images_in_state = True diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 952c874bc8e..f77474d2872 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import Final, Literal from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type, standard_base64 from litellm.types.decisions import ( DecisionsAsset, DecisionsIRMessages, @@ -22,7 +22,6 @@ _IMAGES_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}images" _ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType( {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"} ) -_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") def _image_reference(image_url: str) -> str: @@ -36,7 +35,7 @@ def _image_asset(image_url: str) -> DecisionsAsset | None: return DecisionsAsset( type="image", format=asset_format, - data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64), + data_base64=standard_base64(image_url.partition(",")[2]), ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index a7dd9d90898..6df93598e80 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -4124,21 +4124,93 @@ def test_systemone_moves_images_into_state( model=model, state={"ticket": 1234}, questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - images=["data:IMAGE/JPEG;name=shot.jpg;base64,AA==", {"content_type": "image/webp", "base64": "BB=="}], + images=["data:IMAGE/JPEG;name=shot.jpg;base64,A-_A", {"content_type": "image/webp", "base64": "B-_B"}], api_key="caller-key", ) assert json.loads(route.calls.last.request.content) == { "model": upstream_model, "state": [ - '{"ticket": 1234}', - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AA=="}}, - {"type": "image_url", "image_url": {"url": "data:image/webp;base64,BB=="}}, + {"ticket": 1234}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/A"}}, + {"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}}, ], "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_a_list_state_keeps_its_items_ahead_of_the_images( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=["Ticket 1234", {"severity": "high"}], + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + "Ticket 1234", + {"severity": "high"}, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ] + + +@pytest.mark.parametrize( + ("model", "url", "max_images"), + ( + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8), + ("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4), + ), + ids=("perplexity", "openrouter-clef"), +) +def test_more_images_than_the_model_takes_are_refused_before_http( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * max_images, + api_key="k", + ) + with pytest.raises( + litellm.BadRequestError, + match=re.escape(f"takes at most {max_images} images per request, got {max_images + 1}"), + ): + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * (max_images + 1), + api_key="k", + ) + + assert route.call_count == 1 + + +def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8) + + with pytest.raises(litellm.BadRequestError, match=re.escape("openrouter takes images of at most 4,194,304 bytes")): + litellm.systemone( + model="openrouter/cloudflare/clef-flash", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[over_four_mib], + api_key="k", + ) + + assert not route.called + + @pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) def test_a_gif_to_a_state_image_provider_is_rejected_before_http( model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch