diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 3b73f65c917..35ae4b42316 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -2,6 +2,7 @@ import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial +from itertools import accumulate from types import MappingProxyType from typing import Final, Literal, TypeAlias @@ -301,7 +302,7 @@ def _unsupported_params( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) refused_image_param: Final = ( - None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + None if provider_config.supports_images_in(model, ir_request.input, headers) else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions @@ -332,6 +333,7 @@ def _is_not_box_question(question: DecisionsIRQuestion) -> bool: def _without_unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, + model: str, headers: Mapping[str, str], supports_box_questions: bool, ) -> DecisionsIRRequest: @@ -339,7 +341,7 @@ def _without_unsupported_params( ir_request, input=( ir_request.input - if provider_config.supports_images_in(ir_request.input, headers) + if provider_config.supports_images_in(model, ir_request.input, headers) else with_images_where(ir_request.input, _no_image) ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, @@ -400,7 +402,7 @@ def _provider_ir_request( return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params(ir_request, provider_config, model, headers, supports_box_questions) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -583,9 +585,9 @@ def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]: return remote_image_urls(call.ir_request.input) -def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall: +def _with_images_replaced(call: DecisionsCall, replacements: Mapping[str, str]) -> DecisionsCall: return replace( - call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls)) + call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, replacements)) ) @@ -593,7 +595,7 @@ def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall: remote_urls: Final = _remote_image_urls(call) if not remote_urls: return call - return _with_data_urls( + return _with_images_replaced( call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls}) ) @@ -603,7 +605,7 @@ async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCal if not remote_urls: return call data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout) - return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))) + return _with_images_replaced(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))) def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool: @@ -612,7 +614,7 @@ def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bo def _checked_image_types(call: DecisionsCall) -> DecisionsCall: - media_types: Final = call.provider_config.image_media_types + media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types if media_types is None: return call found: Final = (image_media_type(image) for image in input_images(call.ir_request.input)) @@ -621,28 +623,41 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall: ) if not unsupported: return call - supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)) - if _drops_params(call.litellm_params) and not _leaves_no_input(supported): - return replace(call, ir_request=replace(call.ir_request, input=supported)) + return _dropped_or_refused( + call, + with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)), + f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}", + ) + + +def _with_images_kept( + decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool] +) -> DecisionsIRState | DecisionsIRMessages: + flags: Final = iter(kept) + return with_images_where(decision_input, lambda _image: next(flags)) + + +def _dropped_or_refused( + call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str +) -> DecisionsCall: + if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input): + return replace(call, ir_request=replace(call.ir_request, input=kept_input)) raise litellm.BadRequestError( - message=( - f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, " - f"got {', '.join(unsupported)}" - ), + message=f"{call.custom_llm_provider} {refusal}", model=call.requested_model, llm_provider=call.custom_llm_provider, ) def _checked_image_count(call: DecisionsCall) -> DecisionsCall: - max_images: Final = call.provider_config.max_images + max_images: Final = call.provider_config.image_rules_for(call.model).max_images image_count: Final = len(input_images(call.ir_request.input)) if max_images is None or image_count <= max_images: return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, + return _dropped_or_refused( + call, + _with_images_kept(call.ir_request.input, tuple(index < max_images for index in range(image_count))), + f"takes at most {max_images} images per request, got {image_count}", ) @@ -657,11 +672,8 @@ def _byte_limit(limit: int) -> str: return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)" -def _image_size_error(call: DecisionsCall) -> str | None: - sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) +def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None: largest: Final = max(sizes, default=0) - per_image: Final = call.provider_config.max_image_bytes - total: Final = call.provider_config.max_total_image_bytes if per_image is not None and largest > per_image: return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes" if total is not None and sum(sizes) > total: @@ -669,14 +681,26 @@ def _image_size_error(call: DecisionsCall) -> str | None: return None +def _kept_within_sizes(sizes: Sequence[int], per_image: int | None, total: int | None) -> tuple[bool, ...]: + fits: Final = tuple(per_image is None or size <= per_image for size in sizes) + spent: Final = accumulate(size if fit else 0 for size, fit in zip(sizes, fits, strict=True)) + return tuple( + fit and (total is None or bytes_so_far <= total) for fit, bytes_so_far in zip(fits, spent, strict=True) + ) + + def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall: - error: Final = _image_size_error(call) + sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) + rules: Final = call.provider_config.image_rules_for(call.model) + error: Final = _image_size_error(sizes, rules.max_image_bytes, rules.max_total_image_bytes) if error is None: return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} {error}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, + return _dropped_or_refused( + call, + _with_images_kept( + call.ir_request.input, _kept_within_sizes(sizes, rules.max_image_bytes, rules.max_total_image_bytes) + ), + error, ) @@ -693,6 +717,22 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: return body +def _with_caller_urls(sent: DecisionsCall, supported: DecisionsCall, fetched: DecisionsCall) -> DecisionsCall: + pairs: Final = zip(input_images(fetched.ir_request.input), input_images(supported.ir_request.input), strict=True) + caller_urls: Final = MappingProxyType( + {data_url: url for data_url, url in pairs if isinstance(data_url, str) and isinstance(url, str)} + ) + return _with_images_replaced(sent, caller_urls) + + +def _logged_body( + supported: DecisionsCall, fetched: DecisionsCall, sent: DecisionsCall, body: Mapping[str, object] +) -> Mapping[str, object]: + if fetched is supported: + return body + return _request_body(_with_caller_urls(sent, supported, fetched)) + + def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str: returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") if returned is not None and cost_map_key_prices_anything(returned): @@ -756,7 +796,8 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) - inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported))) + fetched: Final = await _async_with_remote_images_inlined(supported) + inlined: Final = _checked_image_sizes(_checked_image_types(fetched)) body: Final = _request_body(inlined) try: response: Final = await _HANDLER.adecisions( @@ -766,7 +807,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon provider_config=inlined.provider_config, request=inlined.ir_request, body=body, - logged_body=body if inlined is supported else _request_body(supported), + logged_body=_logged_body(supported, fetched, inlined, body), api_base=inlined.api_base, api_key=inlined.api_key, headers=inlined.headers, @@ -780,7 +821,8 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) - inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported))) + fetched: Final = _with_remote_images_inlined(supported) + inlined: Final = _checked_image_sizes(_checked_image_types(fetched)) body: Final = _request_body(inlined) try: response: Final = _HANDLER.decisions( @@ -790,7 +832,7 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: provider_config=inlined.provider_config, request=inlined.ir_request, body=body, - logged_body=body if inlined is supported else _request_body(supported), + logged_body=_logged_body(supported, fetched, inlined, body), api_base=inlined.api_base, api_key=inlined.api_key, headers=inlined.headers, diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index e8e4b77e757..e5f63ac4f5a 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -4,7 +4,7 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import replace from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError @@ -65,12 +65,20 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" +_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") +_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"}) + +_StateItem: TypeAlias = str | Mapping[str, object] def decisions_text(value: DecisionsJSON) -> str: return value if isinstance(value, str) else json.dumps(value) +def standard_base64(value: str) -> str: + return value.translate(_URL_SAFE_TO_STANDARD_BASE64) + + def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]: names: Final = tuple(question.name for question in questions if question.name is not None) if len(frozenset(names)) == len(questions): @@ -260,6 +268,53 @@ def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]: return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}" +def _standard_data_url(media_type: str, payload: str) -> str: + padding: Final = "=" * (-len(payload) % 4) + standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type) + return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_base64(payload)}{padding}" + + +def _data_url(image: DecisionsImage) -> str: + if isinstance(image, DecisionsBase64Image): + return _standard_data_url(_media_type(image.content_type), image.base64) + media_type: Final = image_media_type(image) + if media_type is None: + return image + return _standard_data_url(media_type, image.partition(",")[2]) + + +def _image_item(image: DecisionsImage) -> _StateItem: + return {"type": "image_url", "image_url": {"url": _data_url(image)}} + + +def _part_item(part: OpenAIDecisionInputText | OpenAIDecisionInputImage) -> _StateItem: + if isinstance(part, OpenAIDecisionInputText): + return part.text + return _image_item(part.image_url) + + +def _message_items(message: OpenAIDecisionInputMessage) -> tuple[_StateItem, ...]: + if isinstance(message.content, str): + return (message.content,) + return tuple(map(_part_item, message.content)) + + +def _state_items(state: DecisionsJSON) -> tuple[object, ...]: + if isinstance(state, (str, Mapping)): + return (state,) + return tuple(state) + + +def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[object, ...]: + match decision_input: + case DecisionsIRState(): + return (*_state_items(decision_input.state), *map(_image_item, decision_input.images)) + case DecisionsIRMessages(): + return tuple(chain.from_iterable(map(_message_items, decision_input.messages))) + case _: + assert_never(decision_input) + + def _optional(key: str, value: object) -> Mapping[str, object]: return {} if value is None else {key: value} @@ -298,18 +353,23 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: def ir_to_systemone_request( - model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False + model: str, + request: DecisionsIRRequest, + *, + supports_audio: bool = False, + supports_video: bool = False, + images_in_state: bool = False, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: if request.audio and not supports_audio: return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model") if request.videos and not supports_video: return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") - state: Final = _systemone_state(request.input) keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) images: Final = input_images(request.input) + state_carries_images: Final = images_in_state and bool(images) return { "model": model, - "state": state, + "state": _multimodal_state(request.input) if state_carries_images else _systemone_state(request.input), **( {"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]} if request.audio @@ -321,7 +381,7 @@ def ir_to_systemone_request( else {} ), "questions": {key: _systemone_question(question) for key, question in keyed_questions}, - **({"images": [_systemone_image(image) for image in images]} if images else {}), + **({"images": [_systemone_image(image) for image in images]} if images and not state_carries_images else {}), **( {"assets": {key: asset.model_dump(exclude_none=True) for key, asset in request.assets.items()}} if request.assets @@ -523,6 +583,7 @@ class BaseDecisionsConfig(ABC): max_total_image_bytes: int | None = None image_media_types: frozenset[str] | None = None inlines_remote_images: bool = False + images_in_state: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) @@ -605,10 +666,13 @@ class BaseDecisionsConfig(ABC): return headers, None def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return self.supports_images + def image_rules_for(self, model: str) -> "BaseDecisionsConfig": + return self + def supports_assets(self, headers: Mapping[str, str]) -> bool: return False @@ -642,6 +706,7 @@ class BaseDecisionsConfig(ABC): request, supports_audio=self.supports_audio_input(model, custom_llm_provider), supports_video=self.supports_video_input(model, custom_llm_provider), + images_in_state=self.images_in_state, ) def unwrap_response(self, payload: object) -> object: diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 30d33f09efd..86573580181 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,17 +1,44 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.types.decisions import DecisionsIRResponse +from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig +from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig +from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig +from litellm.types.decisions import DecisionsIRMessages, DecisionsIRResponse, DecisionsIRState + +_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( + { + "typesafe": TypeSafeDecisionsConfig(), + "cloudflare": CloudflareDecisionsConfig(), + "perplexity": PerplexityDecisionsConfig(), + } +) + + +def _upstream_org(model: str) -> str: + return model.removeprefix("~").partition("/")[0] class OpenRouterDecisionsConfig(BaseDecisionsConfig): path = "/alpha/decisions" api_key_env = ("OPENROUTER_API_KEY",) api_base_env = ("OPENROUTER_API_BASE",) + inlines_remote_images = True + images_in_state = True def get_default_api_base(self) -> str | None: return "https://openrouter.ai/api" + def supports_images_in( + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + ) -> bool: + return _upstream_org(model) in _ORG_IMAGE_RULES + + def image_rules_for(self, model: str) -> BaseDecisionsConfig: + return _ORG_IMAGE_RULES.get(_upstream_org(model), self) + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: cost: Final = response.usage.extra.get("cost") if response.usage is not None else None if not isinstance(cost, (int, float)): diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 11e2a38f06e..06989ba4134 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -5,6 +5,11 @@ class PerplexityDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") api_base_env = ("PERPLEXITY_API_BASE",) + supports_images = True + max_images = 8 + image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) + inlines_remote_images = True + images_in_state = True def get_default_api_base(self) -> str | None: return "https://api.perplexity.ai" diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 952c874bc8e..3ba6504c576 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import Final, Literal from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type, standard_base64 from litellm.types.decisions import ( DecisionsAsset, DecisionsIRMessages, @@ -22,7 +22,6 @@ _IMAGES_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}images" _ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType( {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"} ) -_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") def _image_reference(image_url: str) -> str: @@ -36,7 +35,7 @@ def _image_asset(image_url: str) -> DecisionsAsset | None: return DecisionsAsset( type="image", format=asset_format, - data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64), + data_base64=standard_base64(image_url.partition(",")[2]), ) @@ -85,7 +84,7 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig): return "https://api.typesafe.ai" def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..6248e9bf227 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45868,6 +45868,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..6248e9bf227 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45868,6 +45868,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 2f82561d1b1..c88d26db599 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -162,6 +162,7 @@ _PROVIDERS: Final = ( _API_KEY, False, "perplexity/pplx-decider-v1-27b", + takes_images=True, ), _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), _Provider( @@ -172,6 +173,7 @@ _PROVIDERS: Final = ( _API_KEY, False, None, + takes_images=True, provider_reported_cost=1.5834e-5, ), _Provider( @@ -215,9 +217,10 @@ _PROVIDERS: Final = ( _SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) _TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images) _PERPLEXITY: Final = _PROVIDERS[0] +_OPENROUTER: Final = next(provider for provider in _PROVIDERS if provider.name == "openrouter") _TYPESAFE: Final = _PROVIDERS[1] _OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) -_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images) +_CLOUDFLARE: Final = next(provider for provider in _PROVIDERS if provider.name == "cloudflare") _PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"input": _INPUT}), @@ -510,6 +513,21 @@ def test_cloudflare_receives_images_from_either_format_in_its_images_field(gatew assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2 +@pytest.mark.parametrize("provider", (_PERPLEXITY, _OPENROUTER), ids=_provider_id) +def test_perplexity_and_openrouter_receive_images_from_either_format_in_their_state( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model) + assert openai_format.status_code == 200, openai_format.text + assert system_one_format.status_code == 200, system_one_format.text + image_item: Final[dict[str, JsonValue]] = {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}} + expected_body: Final = {**provider.upstream_body(), "state": [_INPUT, image_item]} + assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2 + + def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = _register(scenario, _OPENAI.upstream_reply()) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e5501310556..857ec840d05 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1913,6 +1913,8 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_ _CLOUDFLARE_RUN_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare" _PERPLEXITY_DECISIONS_URL: Final = "https://api.perplexity.ai/v1/decisions" +_TEXT_ONLY_MODEL: Final = "nace/drex-v2.0" +_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone" _PNG_DATA_URL: Final = "data:image/png;base64,AA==" _REMOTE_IMAGE_URL: Final = "https://images.example/screen.png" _SCREEN_TEXT: Final = "Is the screen cracked?" @@ -2119,10 +2121,10 @@ def test_images_are_refused_before_http_when_the_provider_cannot_take_them( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: - getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) + getattr(litellm, api)(model=_TEXT_ONLY_MODEL, api_key="caller-key", **request_kwargs) assert caught.value.status_code == 400 assert not route.called @@ -2134,14 +2136,12 @@ async def test_images_are_dropped_from_the_wire_under_drop_params( api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) - await getattr(litellm, f"a{api}")( - model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs - ) + await getattr(litellm, f"a{api}")(model=_TEXT_ONLY_MODEL, api_key="caller-key", drop_params=True, **request_kwargs) assert json.loads(route.calls[0].request.content) == { - "model": "pplx-decider-v1-27b", + "model": "drex-v2.0", "state": _SCREEN_TEXT, "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } @@ -2307,7 +2307,7 @@ def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, obje def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http( api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(litellm, "drop_params", True) + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: @@ -2334,11 +2334,11 @@ async def test_dropping_the_only_input_images_is_rejected_before_http( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=r"\['input_image'\].*leaves no input") as caught: await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", + model=_TEXT_ONLY_MODEL, api_key="caller-key", input=[{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}], questions=_PREDICATE_QUESTIONS, @@ -2445,6 +2445,10 @@ def _png_object(decoded_bytes: int) -> Mapping[str, object]: return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} +def _distinct_png_data_urls(count: int) -> tuple[str, ...]: + return tuple(f"data:image/png;base64,{base64.b64encode(bytes((index,))).decode()}" for index in range(count)) + + _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( ((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), ((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), @@ -2455,29 +2459,111 @@ _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( ) +_OVERSIZED_IMAGE_IDS: Final = ("data_url_over_4mib", "object_over_4mib", "total_over_8mib") + + @pytest.mark.usefixtures("cloudflare_account") -@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) -@pytest.mark.parametrize( - ("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib") -) +@pytest.mark.parametrize(("images", "refusal"), _OVERSIZED_IMAGES, ids=_OVERSIZED_IMAGE_IDS) def test_images_over_cloudflares_size_limits_are_refused_before_http( - images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter + images: Sequence[object], refusal: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught: - litellm.systemone( - model="cloudflare/clef", - state=_SCREEN_TEXT, - questions=_SYSTEMONE_PREDICATE, - images=images, - drop_params=drop_params, - ) + litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images) assert caught.value.status_code == 400 assert not route.called +_SMALL_PNG: Final = _png_data_url(1) +_DROPPED_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], Sequence[object]], ...]] = ( + *((images, images[:kept]) for (images, _), kept in zip(_OVERSIZED_IMAGES, (0, 0, 2), strict=True)), + ((_png_data_url(4 * _MIB + 1), _SMALL_PNG), (_SMALL_PNG,)), +) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize( + ("images", "kept"), _DROPPED_OVERSIZED_IMAGES, ids=(*_OVERSIZED_IMAGE_IDS, "oversized_then_small") +) +def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params( + images: Sequence[object], kept: Sequence[object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=images, + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content).get("images", []) == list(kept) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_logging_callbacks_get_only_the_images_left_after_an_oversized_one_drops( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_png_data_url(4 * _MIB + 1), _SMALL_PNG], + drop_params=True, + ) + await _drain_logging_worker() + + assert recording_logger.inputs == [json.loads(route.calls.last.request.content)] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_logging_callbacks_get_the_caller_url_of_a_remote_image_kept_after_an_oversized_one_drops( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, async_only_image_fetch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_png_data_url(4 * _MIB + 1), _REMOTE_IMAGE_URL], + drop_params=True, + ) + await _drain_logging_worker() + + wire_body: Final = json.loads(route.calls.last.request.content) + assert wire_body["images"] == [async_only_image_fetch.data_url] + assert recording_logger.inputs == [{**wire_body, "images": [_REMOTE_IMAGE_URL]}] + + +@pytest.mark.usefixtures("cloudflare_account") +def test_images_past_cloudflares_limit_are_dropped_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(5) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["images"] == list(images[:4]) + + @pytest.mark.asyncio @pytest.mark.usefixtures("cloudflare_account") async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None: @@ -3588,6 +3674,47 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_ assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest")) +async def test_openrouter_jev_sends_box_questions_and_returns_box_answers( + model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}} + ) + + response: Final = await litellm.adecisions( + model=model, + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], + } + ], + questions=[_CRACK_BOX_QUESTION], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["questions"] == { + "crack": {"type": "box", "instructions": "the crack on the screen"} + } + assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] + + +def test_openrouter_jev_1_13_still_refuses_box_questions( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"): + litellm.decisions( + model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k" + ) + + assert not route.called + + @pytest.mark.asyncio async def test_a_deployment_that_turns_box_questions_off_is_refused_even_when_the_cost_map_allows_them( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch @@ -4253,6 +4380,260 @@ async def test_hosted_vllm_decisions_sends_input_images_to_systemone( assert async_only_image_fetch.fetched == (["https://images.example/input.png"] if remote_image else []) +_STATE_IMAGE_PROVIDERS: Final = ( + ("openrouter/cloudflare/clef", "https://openrouter.ai/api/alpha/decisions", "cloudflare/clef"), + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", "pplx-decider-v1.2"), +) +_STATE_IMAGE_PROVIDER_IDS: Final = ("openrouter", "perplexity") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +async def test_decisions_sends_input_images_in_state_order( + model: str, + url: str, + upstream_model: str, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + image: Final[str] = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA==" + expected_image: Final[str] = async_only_image_fetch.data_url if remote_image else image + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model=model, + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Listing photo:"}, + {"type": "input_image", "image_url": image}, + {"type": "input_text", "text": "Is it damaged?"}, + ], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": upstream_model, + "state": [ + "Listing photo:", + {"type": "image_url", "image_url": {"url": expected_image}}, + "Is it damaged?", + ], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].probability == 0.9 + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_systemone_moves_images_into_state( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state={"ticket": 1234}, + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + images=["data:IMAGE/JPG;name=shot.jpg;base64,A-_AB-", {"content_type": "image/webp", "base64": "B-_B"}], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": upstream_model, + "state": [ + {"ticket": 1234}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/AB+=="}}, + {"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}}, + ], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_a_list_state_keeps_its_items_ahead_of_the_images( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=["Ticket 1234", {"severity": "high"}], + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + "Ticket 1234", + {"severity": "high"}, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ] + + +_STATE_IMAGE_LIMITS: Final = ( + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8), + ("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4), + ("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions", 8), +) +_STATE_IMAGE_LIMIT_IDS: Final = ("perplexity", "openrouter-clef", "openrouter-perplexity") + + +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) +def test_more_images_than_the_model_takes_are_refused_before_http( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * max_images, + api_key="k", + ) + with pytest.raises( + litellm.BadRequestError, + match=re.escape(f"takes at most {max_images} images per request, got {max_images + 1}"), + ): + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * (max_images + 1), + api_key="k", + ) + + assert route.call_count == 1 + + +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) +def test_images_past_the_models_limit_are_dropped_under_drop_params( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(max_images + 2) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + api_key="k", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + *({"type": "image_url", "image_url": {"url": image}} for image in images[:max_images]), + ] + + +def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8) + + with pytest.raises(litellm.BadRequestError, match=re.escape("openrouter takes images of at most 4,194,304 bytes")): + litellm.systemone( + model="openrouter/cloudflare/clef-flash", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[over_four_mib], + api_key="k", + ) + + assert not route.called + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_a_gif_to_a_state_image_provider_is_rejected_before_http( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + with pytest.raises( + litellm.BadRequestError, match=re.escape("takes only image/jpeg, image/jpg, image/png, image/webp images") + ) as caught: + litellm.systemone( + model=model, state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_GIF_DATA_URL], api_key="k" + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest")) +def test_openrouter_jev_rejects_webp_before_http( + model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=re.escape("got image/webp")): + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=["data:image/webp;base64,BB=="], + api_key="k", + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_images_to_an_openrouter_upstream_without_image_rules_are_refused_or_dropped_as_before( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + request: Final[Mapping[str, object]] = MappingProxyType( + { + "model": "openrouter/upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + "images": [_SMALL_PNG], + "api_key": "caller-key", + } + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['images'\]"): + await litellm.asystemone(**request) + await litellm.asystemone(**request, drop_params=True) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == { + "model": "upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + } + + +def test_openrouter_jev_drops_webp_and_keeps_png_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + litellm.systemone( + model="openrouter/typesafe/jev-1.14", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=["data:image/webp;base64,BB==", "data:image/png;base64,AA=="], + api_key="k", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, + ] + + def test_hosted_vllm_systemone_forwards_images( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..b202de567fa 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -962,6 +962,8 @@ _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = ( ) _IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True)) _IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format") +_TEXT_ONLY_DEPLOYMENT: Final[Mapping[str, object]] = {"model": "nace/drex-v2.0", "api_key": "test-key"} +_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone" def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None: @@ -1017,7 +1019,8 @@ def test_images_to_a_text_only_deployment_are_refused_without_drop_params( param: str, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + _route_decider_to(monkeypatch, _TEXT_ONLY_DEPLOYMENT) + upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) response: Final = client.post(endpoint, json=request_body) @@ -1035,16 +1038,14 @@ def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params( request_body: Mapping[str, object], ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - _route_decider_to( - monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True} - ) - upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + _route_decider_to(monkeypatch, {**_TEXT_ONLY_DEPLOYMENT, "drop_params": True}) + upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) response: Final = client.post(endpoint, json=request_body) assert response.status_code == 200, response.text assert json.loads(upstream.calls[0].request.content) == { - "model": "pplx-decider-v1-27b", + "model": "drex-v2.0", "state": _IMAGE_TEXT, "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } @@ -1104,7 +1105,12 @@ def test_malformed_assets_for_a_deployment_without_asset_support_are_dropped_onl response: Final = client.post( "/v1/systemone", - json={**_IMAGE_REQUEST_BODIES[1], "assets": {"photo-1": {"type": "image", "part": "file1"}}}, + json={ + "model": "decider", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "assets": {"photo-1": {"type": "image", "part": "file1"}}, + }, ) if not deployment_drops_params: