diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 9c0e51e14ac..fb62bb2a71d 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,7 +1,9 @@ +import base64 import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial +from itertools import accumulate, islice from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias @@ -18,7 +20,9 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpe from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_urls_to_base64, + async_convert_urls_to_base64_or_errors, convert_url_to_base64, + convert_url_to_base64_or_error, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( @@ -27,9 +31,11 @@ from litellm.llms.base_llm.decisions.transformation import ( catalog_mode, image_media_type, image_param, + input_has_no_text, input_images, ir_to_systemone_response, remote_image_urls, + standard_base64, systemone_request_to_ir, with_images_where, with_inlined_images, @@ -87,6 +93,7 @@ _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ ] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None) _HANDLER: Final = BaseLLMHTTPHandler() _BASE64_MARKER: Final = ";base64," +_SNIFFABLE_MEDIA_TYPES: Final = frozenset({"image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp"}) _IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images")) @@ -380,6 +387,7 @@ class _MalformedImages: def _unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, + model: str, headers: Mapping[str, str], support: _ResolvedSupport, ) -> tuple[str, ...]: @@ -387,7 +395,7 @@ def _unsupported_params( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) refused_image_param: Final = ( - None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + None if provider_config.supports_images_in(model, ir_request.input, headers) else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions @@ -414,6 +422,7 @@ def _is_not_box_question(question: DecisionsIRQuestion) -> bool: def _without_unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, + model: str, headers: Mapping[str, str], support: _ResolvedSupport, ) -> DecisionsIRRequest: @@ -421,7 +430,7 @@ def _without_unsupported_params( ir_request, input=( ir_request.input - if provider_config.supports_images_in(ir_request.input, headers) + if provider_config.supports_images_in(model, ir_request.input, headers) else with_images_where(ir_request.input, _no_image) ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, @@ -452,16 +461,17 @@ def _provider_ir_request( ir_request: DecisionsIRRequest, *, provider_config: BaseDecisionsConfig, + model: str, headers: Mapping[str, str], support: _ResolvedSupport, kwargs: Mapping[str, object], ) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop: - unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support) + unsupported: Final = _unsupported_params(ir_request, provider_config, model, headers, support) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support) + supported: Final = _without_unsupported_params(ir_request, provider_config, model, headers, support) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -585,7 +595,12 @@ def prepare_call( llm_provider=provider, ) ir_request: Final = _provider_ir_request( - caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs + caller_ir_request, + provider_config=provider_config, + model=canonical_model, + headers=headers, + support=support, + kwargs=kwargs, ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( @@ -643,9 +658,38 @@ def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]: return remote_image_urls(call.ir_request.input) -def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall: +def _with_images_replaced(call: DecisionsCall, replacements: Mapping[str, str]) -> DecisionsCall: return replace( - call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls)) + call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, replacements)) + ) + + +def _is_not_one_of(urls: frozenset[str], image: DecisionsImage) -> bool: + return not isinstance(image, str) or image not in urls + + +def _leaves_nothing_to_decide_on(call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages) -> bool: + if _leaves_no_input(kept_input): + return True + return ( + not call.provider_config.images_in_state + and not call.ir_request.audio + and not call.ir_request.videos + and not input_images(kept_input) + and input_has_no_text(kept_input) + ) + + +def _with_downloads_or_dropped_failures( + call: DecisionsCall, downloads: Mapping[str, str | litellm.ImageFetchError] +) -> DecisionsCall: + failures: Final = tuple(result for result in downloads.values() if isinstance(result, litellm.ImageFetchError)) + failed_urls: Final = frozenset(url for url, result in downloads.items() if not isinstance(result, str)) + kept_input: Final = with_images_where(call.ir_request.input, partial(_is_not_one_of, failed_urls)) + if failures and _leaves_nothing_to_decide_on(call, kept_input): + raise failures[0] + return _with_images_replaced( + call, MappingProxyType({url: result for url, result in downloads.items() if isinstance(result, str)}) ) @@ -653,7 +697,12 @@ def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall: remote_urls: Final = _remote_image_urls(call) if not remote_urls: return call - return _with_data_urls( + if _drops_params(call.litellm_params): + return _with_downloads_or_dropped_failures( + call, + MappingProxyType({url: convert_url_to_base64_or_error(url, timeout=call.timeout) for url in remote_urls}), + ) + return _with_images_replaced( call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls}) ) @@ -662,8 +711,13 @@ async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCal remote_urls: Final = _remote_image_urls(call) if not remote_urls: return call + if _drops_params(call.litellm_params): + downloads: Final = await async_convert_urls_to_base64_or_errors(remote_urls, timeout=call.timeout) + return _with_downloads_or_dropped_failures( + call, MappingProxyType(dict(zip(remote_urls, downloads, strict=True))) + ) data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout) - return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))) + return _with_images_replaced(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))) def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool: @@ -671,57 +725,130 @@ def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bo return media_type is None or media_type in media_types -def _checked_image_types(call: DecisionsCall) -> DecisionsCall: - media_types: Final = call.provider_config.image_media_types +def _within(limit: int | None, amount: int) -> bool: + return limit is None or amount <= limit + + +def _media_type_error(media_types: frozenset[str] | None, images: Sequence[DecisionsImage]) -> str | None: if media_types is None: - return call - found: Final = (image_media_type(image) for image in input_images(call.ir_request.input)) + return None + found: Final = (image_media_type(image) for image in images) unsupported: Final = tuple( dict.fromkeys(type_ for type_ in found if type_ is not None and type_ not in media_types) ) if not unsupported: - return call - supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)) - if _drops_params(call.litellm_params) and not _leaves_no_input(supported): - return replace(call, ir_request=replace(call.ir_request, input=supported)) - raise litellm.BadRequestError( - message=( - f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, " - f"got {', '.join(unsupported)}" - ), + return None + return f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}" + + +def _count_error(max_images: int | None, image_count: int) -> str | None: + if _within(max_images, image_count): + return None + return f"takes at most {max_images} images per request, got {image_count}" + + +def _refusal(call: DecisionsCall, refusal: str) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"{call.custom_llm_provider} {refusal}", model=call.requested_model, llm_provider=call.custom_llm_provider, ) -def _checked_image_count(call: DecisionsCall) -> DecisionsCall: - max_images: Final = call.provider_config.max_images - image_count: Final = len(input_images(call.ir_request.input)) - if max_images is None or image_count <= max_images: +def _with_images_kept( + decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool] +) -> DecisionsIRState | DecisionsIRMessages: + flags: Final = iter(kept) + return with_images_where(decision_input, lambda _image: next(flags)) + + +def _dropped_or_refused( + call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str +) -> DecisionsCall: + if _drops_params(call.litellm_params) and not _leaves_nothing_to_decide_on(call, kept_input): + return replace(call, ir_request=replace(call.ir_request, input=kept_input)) + raise _refusal(call, refusal) + + +def _checked_image_types(call: DecisionsCall) -> DecisionsCall: + media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types + error: Final = _media_type_error(media_types, input_images(call.ir_request.input)) + if media_types is None or error is None: return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, + kept_input: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)) + if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input): + return replace(call, ir_request=replace(call.ir_request, input=kept_input)) + raise _refusal(call, error) + + +def _refused_over_count(call: DecisionsCall) -> DecisionsCall: + max_images: Final = call.provider_config.image_rules_for(call.model).max_images + error: Final = _count_error(max_images, len(input_images(call.ir_request.input))) + if error is None: + return call + past_the_count: Final = frozenset(_remote_image_urls(call)[max_images:]) + return _dropped_or_refused( + call, with_images_where(call.ir_request.input, partial(_is_not_one_of, past_the_count)), error ) +def _decoded_byte_count(payload: str) -> int: + return len(standard_base64(payload).rstrip("=")) * 3 // 4 + + +def _base64_payload(image: DecisionsImage) -> str | None: + if isinstance(image, DecisionsBase64Image): + return image.base64 + marker_at: Final = image.lower().find(_BASE64_MARKER) + return None if marker_at < 0 else image[marker_at + len(_BASE64_MARKER) :] + + def _decoded_image_bytes(image: DecisionsImage) -> int: - if isinstance(image, DecisionsBase64Image): - return len(image.base64.rstrip("=")) * 3 // 4 - marker_at: Final = image.lower().find(_BASE64_MARKER) - return 0 if marker_at < 0 else len(image[marker_at + len(_BASE64_MARKER) :].rstrip("=")) * 3 // 4 + payload: Final = _base64_payload(image) + return 0 if payload is None else _decoded_byte_count(payload) + + +def _leading_bytes(payload: str) -> bytes: + head: Final = standard_base64(payload[:64])[:16] + try: + return base64.b64decode(head + "=" * (-len(head) % 4)) + except ValueError: + return b"" + + +def _sniffed_media_types(head: bytes) -> frozenset[str]: + if head.startswith(b"\x89PNG\r\n\x1a\n"): + return frozenset({"image/png"}) + if head.startswith(b"\xff\xd8\xff"): + return frozenset({"image/jpeg", "image/jpg"}) + if head.startswith((b"GIF87a", b"GIF89a")): + return frozenset({"image/gif"}) + if head.startswith(b"RIFF") and head[8:12] == b"WEBP": + return frozenset({"image/webp"}) + return frozenset() + + +def _is_readable_image(media_types: frozenset[str] | None, image: DecisionsImage) -> bool: + payload: Final = _base64_payload(image) + if media_types is None or not media_types <= _SNIFFABLE_MEDIA_TYPES or payload is None: + return True + return bool(_sniffed_media_types(_leading_bytes(payload)) & media_types) + + +def _unreadable_image_error(call: DecisionsCall) -> str | None: + media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types + unreadable: Final = sum(not _is_readable_image(media_types, image) for image in input_images(call.ir_request.input)) + if media_types is None or not unreadable: + return None + return f"takes only {', '.join(sorted(media_types))} image bytes, got {unreadable} image(s) that are none of these" def _byte_limit(limit: int) -> str: return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)" -def _image_size_error(call: DecisionsCall) -> str | None: - sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) +def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None: largest: Final = max(sizes, default=0) - per_image: Final = call.provider_config.max_image_bytes - total: Final = call.provider_config.max_total_image_bytes if per_image is not None and largest > per_image: return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes" if total is not None and sum(sizes) > total: @@ -729,17 +856,79 @@ def _image_size_error(call: DecisionsCall) -> str | None: return None -def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall: - error: Final = _image_size_error(call) - if error is None: - return call - raise litellm.BadRequestError( - message=f"{call.custom_llm_provider} {error}", - model=call.requested_model, - llm_provider=call.custom_llm_provider, +def _image_limit_error(call: DecisionsCall) -> str | None: + rules: Final = call.provider_config.image_rules_for(call.model) + images: Final = input_images(call.ir_request.input) + return ( + _media_type_error(rules.image_media_types, images) + or _count_error(rules.max_images, len(images)) + or _image_size_error( + tuple(_decoded_image_bytes(image) for image in images), rules.max_image_bytes, rules.max_total_image_bytes + ) ) +def _fits_alone(rules: BaseDecisionsConfig, image: DecisionsImage) -> bool: + return (rules.image_media_types is None or _has_media_type_in(rules.image_media_types, image)) and _within( + rules.max_image_bytes, _decoded_image_bytes(image) + ) + + +def _is_usable( + rules: BaseDecisionsConfig, failed_downloads: frozenset[str], drops_params: bool, image: DecisionsImage +) -> bool: + return ( + _is_not_one_of(failed_downloads, image) + and _fits_alone(rules, image) + and (not drops_params or _is_readable_image(rules.image_media_types, image)) + ) + + +@dataclass(frozen=True, slots=True) +class _ImageBudget: + spent: int = 0 + count: int = 0 + kept: bool = False + + +def _after_image(rules: BaseDecisionsConfig, budget: _ImageBudget, sized_fit: tuple[int, bool]) -> _ImageBudget: + size, fit = sized_fit + if not fit or not _within(rules.max_total_image_bytes, budget.spent + size): + return replace(budget, kept=False) + if not _within(rules.max_images, budget.count + 1): + return replace(budget, kept=False) + return _ImageBudget(spent=budget.spent + size, count=budget.count + 1, kept=True) + + +def _kept_images(call: DecisionsCall) -> tuple[bool, ...]: + rules: Final = call.provider_config.image_rules_for(call.model) + images: Final = input_images(call.ir_request.input) + usable: Final = partial(_is_usable, rules, frozenset(_remote_image_urls(call)), _drops_params(call.litellm_params)) + budgets: Final = accumulate( + zip(map(_decoded_image_bytes, images), map(usable, images), strict=True), + partial(_after_image, rules), + initial=_ImageBudget(), + ) + return tuple(budget.kept for budget in islice(budgets, 1, None)) + + +def _drops_images(before: DecisionsCall, after: DecisionsCall) -> bool: + return len(input_images(after.ir_request.input)) < len(input_images(before.ir_request.input)) + + +def _with_images_within_limits(call: DecisionsCall, kept: Sequence[bool], dropped_past_count: bool) -> DecisionsCall: + if all(kept): + return call + kept_input: Final = _with_images_kept(call.ir_request.input, kept) + error: Final = _image_limit_error(call) or _unreadable_image_error(call) + if error is None: + return replace(call, ir_request=replace(call.ir_request, input=kept_input)) + type_checked: Final = _checked_image_types(call) + if not dropped_past_count and input_images(type_checked.ir_request.input) == input_images(kept_input): + return type_checked + return _dropped_or_refused(call, kept_input, error) + + def _request_body(call: DecisionsCall) -> Mapping[str, object]: body: Final = call.provider_config.transform_decisions_request( model=call.model, request=call.ir_request, custom_llm_provider=call.custom_llm_provider @@ -753,6 +942,19 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: return body +def _logged_body( + supported: DecisionsCall, fetched: DecisionsCall, kept: Sequence[bool], body: Mapping[str, object] +) -> Mapping[str, object]: + if fetched is supported: + return body + return _request_body( + replace( + supported, + ir_request=replace(supported.ir_request, input=_with_images_kept(supported.ir_request.input, kept)), + ) + ) + + def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str: returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") if returned is not None and cost_map_key_prices_anything(returned): @@ -816,24 +1018,105 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: ) +_RESENT_WITHOUT_IMAGES_STATUSES: Final = frozenset({400, 408, 413, 422}) + + +def _may_be_caused_by_images(error: BaseLLMException | litellm.Timeout) -> bool: + if isinstance(error, litellm.Timeout): + return True + return not error.status_code_is_synthesized and error.status_code in _RESENT_WITHOUT_IMAGES_STATUSES + + +def _goes_again_without_images(call: DecisionsCall, error: BaseLLMException | litellm.Timeout) -> bool: + return ( + call.provider_config.images_in_state and _drops_params(call.litellm_params) and _may_be_caused_by_images(error) + ) + + +def _without_images_after_refusal( + call: DecisionsCall, error: BaseLLMException | litellm.Timeout +) -> DecisionsCall | None: + images: Final = input_images(call.ir_request.input) + if not images or not _goes_again_without_images(call, error): + return None + without_images: Final = _with_images_kept(call.ir_request.input, tuple(False for _ in images)) + if _leaves_no_input(without_images): + return None + return replace(call, ir_request=replace(call.ir_request, input=without_images)) + + +async def _aposted( + call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object] +) -> DecisionsIRResponse: + return await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=body, + logged_body=logged_body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + + +def _posted(call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object]) -> DecisionsIRResponse: + return _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=body, + logged_body=logged_body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + + +async def _aposted_without_refused_images( + call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object] +) -> DecisionsIRResponse: + try: + return await _aposted(call, body, logged_body) + except (BaseLLMException, litellm.Timeout) as error: + retry: Final = _without_images_after_refusal(call, error) + if retry is None: + raise + retry_body: Final = _request_body(retry) + return await _aposted(retry, retry_body, retry_body) + + +def _posted_without_refused_images( + call: DecisionsCall, body: Mapping[str, object], logged_body: Mapping[str, object] +) -> DecisionsIRResponse: + try: + return _posted(call, body, logged_body) + except (BaseLLMException, litellm.Timeout) as error: + retry: Final = _without_images_after_refusal(call, error) + if retry is None: + raise + retry_body: Final = _request_body(retry) + return _posted(retry, retry_body, retry_body) + + async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: - supported: Final = _checked_image_count(_checked_image_types(call)) - inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported))) + type_checked: Final = _checked_image_types(call) + supported: Final = _refused_over_count(type_checked) + fetched: Final = await _async_with_remote_images_inlined(supported) + kept: Final = _kept_images(fetched) + inlined: Final = _with_images_within_limits(fetched, kept, _drops_images(type_checked, supported)) body: Final = _request_body(inlined) try: - response: Final = await _HANDLER.adecisions( - model=inlined.model, - custom_llm_provider=inlined.custom_llm_provider, - logging_obj=inlined.logging_obj, - provider_config=inlined.provider_config, - request=inlined.ir_request, - body=body, - logged_body=body if inlined is supported else _request_body(supported), - api_base=inlined.api_base, - api_key=inlined.api_key, - headers=inlined.headers, - timeout=inlined.timeout, - litellm_params=inlined.litellm_params, + response: Final = await _aposted_without_refused_images( + inlined, body, _logged_body(supported, fetched, kept, body) ) except Exception as error: raise _map_upstream_exception(error, inlined) from error @@ -841,24 +1124,14 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: - supported: Final = _checked_image_count(_checked_image_types(call)) - inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported))) + type_checked: Final = _checked_image_types(call) + supported: Final = _refused_over_count(type_checked) + fetched: Final = _with_remote_images_inlined(supported) + kept: Final = _kept_images(fetched) + inlined: Final = _with_images_within_limits(fetched, kept, _drops_images(type_checked, supported)) body: Final = _request_body(inlined) try: - response: Final = _HANDLER.decisions( - model=inlined.model, - custom_llm_provider=inlined.custom_llm_provider, - logging_obj=inlined.logging_obj, - provider_config=inlined.provider_config, - request=inlined.ir_request, - body=body, - logged_body=body if inlined is supported else _request_body(supported), - api_base=inlined.api_base, - api_key=inlined.api_key, - headers=inlined.headers, - timeout=inlined.timeout, - litellm_params=inlined.litellm_params, - ) + response: Final = _posted_without_refused_images(inlined, body, _logged_body(supported, fetched, kept, body)) except Exception as error: raise _map_upstream_exception(error, inlined) from error return _format_response(response, inlined) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index fa6f1c1ed0a..b593936be36 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -522,6 +522,31 @@ async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) - raise +def convert_url_to_base64_or_error( + url: str, timeout: float | httpx.Timeout | None = None +) -> "str | litellm.ImageFetchError": + try: + return convert_url_to_base64(url, timeout=timeout) + except litellm.ImageFetchError as error: + return error + + +async def _fetch_data_url_or_error( + url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None +) -> "str | litellm.ImageFetchError": + try: + return await _fetch_data_url(url, in_flight, timeout) + except litellm.ImageFetchError as error: + return error + + +async def async_convert_urls_to_base64_or_errors( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> "tuple[str | litellm.ImageFetchError, ...]": + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + return tuple(await asyncio.gather(*(_fetch_data_url_or_error(url, in_flight, timeout) for url in remote_urls))) + + async def async_convert_urls_to_base64( remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None ) -> tuple[str, ...]: diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 254783edf39..b2201773be9 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -4,14 +4,15 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError from pydantic_core import ErrorDetails -from typing_extensions import assert_never +from typing_extensions import TypeIs, assert_never # noqa: TID251 # TypeIs reaches typing only on 3.13 import litellm +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -66,12 +67,20 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" +_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") +_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"}) + +_StateItem: TypeAlias = str | Mapping[str, object] def decisions_text(value: DecisionsJSON) -> str: return value if isinstance(value, str) else json.dumps(value) +def standard_base64(value: str) -> str: + return "".join(value.split()).translate(_URL_SAFE_TO_STANDARD_BASE64) + + def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]: names: Final = tuple(question.name for question in questions if question.name is not None) if len(frozenset(names)) == len(questions): @@ -240,6 +249,34 @@ def _systemone_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> assert_never(decision_input) +def _is_json_object(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: JSON object keys are strings + return isinstance(value, Mapping) + + +def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: only the items are read + return isinstance(value, Sequence) + + +def _holds_no_text(value: object, depth: int = 0) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + if depth >= DEFAULT_MAX_RECURSE_DEPTH: + return True + if _is_json_object(value): + return all( + _holds_no_text(item, depth + 1) for key, item in value.items() if key != "type" or "text" not in value + ) + if _is_json_array(value): + return all(_holds_no_text(item, depth + 1) for item in value) + return False + + +def input_has_no_text(decision_input: DecisionsIRState | DecisionsIRMessages) -> bool: + return _holds_no_text(_systemone_state(decision_input)) + + def _media_type(content_type: str) -> str: return content_type.split(";", 1)[0].strip().lower() @@ -261,6 +298,54 @@ def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]: return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}" +def _standard_data_url(media_type: str, payload: str) -> str: + standard_payload: Final = standard_base64(payload) + padding: Final = "=" * (-len(standard_payload) % 4) + standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type) + return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_payload}{padding}" + + +def _data_url(image: DecisionsImage) -> str: + if isinstance(image, DecisionsBase64Image): + return _standard_data_url(_media_type(image.content_type), image.base64) + media_type: Final = image_media_type(image) + if media_type is None: + return image + return _standard_data_url(media_type, image.partition(",")[2]) + + +def _image_item(image: DecisionsImage) -> _StateItem: + return {"type": "image_url", "image_url": {"url": _data_url(image)}} + + +def _part_item(part: OpenAIDecisionInputText | OpenAIDecisionInputImage) -> _StateItem: + if isinstance(part, OpenAIDecisionInputText): + return part.text + return _image_item(part.image_url) + + +def _message_items(message: OpenAIDecisionInputMessage) -> tuple[_StateItem, ...]: + if isinstance(message.content, str): + return (message.content,) + return tuple(map(_part_item, message.content)) + + +def _state_items(state: DecisionsJSON) -> tuple[object, ...]: + if isinstance(state, (str, Mapping)): + return (state,) + return tuple(state) + + +def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[object, ...]: + match decision_input: + case DecisionsIRState(): + return (*_state_items(decision_input.state), *map(_image_item, decision_input.images)) + case DecisionsIRMessages(): + return tuple(chain.from_iterable(map(_message_items, decision_input.messages))) + case _: + assert_never(decision_input) + + def _optional(key: str, value: object) -> Mapping[str, object]: return {} if value is None else {key: value} @@ -298,13 +383,15 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: assert_never(question) -def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: - state: Final = _systemone_state(request.input) +def ir_to_systemone_request( + model: str, request: DecisionsIRRequest, *, images_in_state: bool = False +) -> Mapping[str, object]: keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) images: Final = input_images(request.input) + state_carries_images: Final = images_in_state and bool(images) return { "model": model, - "state": state, + "state": [*_multimodal_state(request.input)] if state_carries_images else _systemone_state(request.input), **( {"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]} if request.audio @@ -316,7 +403,7 @@ def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[ else {} ), "questions": {key: _systemone_question(question) for key, question in keyed_questions}, - **({"images": [_systemone_image(image) for image in images]} if images else {}), + **({"images": [_systemone_image(image) for image in images]} if images and not state_carries_images else {}), **( {"assets": {key: asset.model_dump(exclude_none=True) for key, asset in request.assets.items()}} if request.assets @@ -538,6 +625,7 @@ class BaseDecisionsConfig(ABC): max_total_image_bytes: int | None = None image_media_types: frozenset[str] | None = None inlines_remote_images: bool = False + images_in_state: bool = False openai_evidence_fields: frozenset[str] = frozenset() confidence_optional: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( @@ -635,10 +723,13 @@ class BaseDecisionsConfig(ABC): return headers, None def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return self.supports_images + def image_rules_for(self, model: str) -> "BaseDecisionsConfig": + return self + def supports_assets(self, headers: Mapping[str, str]) -> bool: return False @@ -671,7 +762,7 @@ class BaseDecisionsConfig(ABC): request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(self.request_model(model), request) + return ir_to_systemone_request(self.request_model(model), request, images_in_state=self.images_in_state) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 30d33f09efd..86573580181 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,17 +1,44 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.types.decisions import DecisionsIRResponse +from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig +from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig +from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig +from litellm.types.decisions import DecisionsIRMessages, DecisionsIRResponse, DecisionsIRState + +_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( + { + "typesafe": TypeSafeDecisionsConfig(), + "cloudflare": CloudflareDecisionsConfig(), + "perplexity": PerplexityDecisionsConfig(), + } +) + + +def _upstream_org(model: str) -> str: + return model.removeprefix("~").partition("/")[0] class OpenRouterDecisionsConfig(BaseDecisionsConfig): path = "/alpha/decisions" api_key_env = ("OPENROUTER_API_KEY",) api_base_env = ("OPENROUTER_API_BASE",) + inlines_remote_images = True + images_in_state = True def get_default_api_base(self) -> str | None: return "https://openrouter.ai/api" + def supports_images_in( + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + ) -> bool: + return _upstream_org(model) in _ORG_IMAGE_RULES + + def image_rules_for(self, model: str) -> BaseDecisionsConfig: + return _ORG_IMAGE_RULES.get(_upstream_org(model), self) + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: cost: Final = response.usage.extra.get("cost") if response.usage is not None else None if not isinstance(cost, (int, float)): diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 11e2a38f06e..06989ba4134 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -5,6 +5,11 @@ class PerplexityDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") api_base_env = ("PERPLEXITY_API_BASE",) + supports_images = True + max_images = 8 + image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) + inlines_remote_images = True + images_in_state = True def get_default_api_base(self) -> str | None: return "https://api.perplexity.ai" diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 871596b7f00..136eb1e955f 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -6,7 +6,12 @@ from types import MappingProxyType from typing import Final, Literal from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, DecisionsInputLimits, image_media_type +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + DecisionsInputLimits, + image_media_type, + standard_base64, +) from litellm.types.decisions import ( DecisionsAsset, DecisionsIRMessages, @@ -23,7 +28,6 @@ _LONG_CONTEXT_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}long-context _ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType( {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"} ) -_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") def _image_reference(image_url: str) -> str: @@ -37,7 +41,7 @@ def _image_asset(image_url: str) -> DecisionsAsset | None: return DecisionsAsset( type="image", format=asset_format, - data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64), + data_base64=standard_base64(image_url.partition(",")[2]), ) @@ -96,7 +100,7 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig): return DecisionsInputLimits(max_request_tokens=65536, max_question_tokens=32768) def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b102d6e5a07..a9f9e6f2f20 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47256,6 +47256,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b102d6e5a07..a9f9e6f2f20 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47256,6 +47256,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index f4ca7fa2965..b06cfbe4a4d 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -51,6 +51,7 @@ IGNORE_FUNCTIONS = [ "_convert_to_json_serializable_dict", # max depth set (default 20) and circular reference protection to prevent infinite recursion. "dict", # max depth set. _LiteLLMParamsDictView.dict() calls builtin dict(), not itself. "_read_image_bytes", # max depth set. + "_holds_no_text", # max depth set. "get_masked_values", # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts. "_redact_sensitive_litellm_params", # max depth set (default 10). "_redact_secret_values_in_obj", # max depth set (default 10, _REDACT_SECRET_MAX_DEPTH); fails closed by returning "REDACTED" at the cap. diff --git a/tests/integration/providers/test_cloudflare_clef_images_wire.py b/tests/integration/providers/test_cloudflare_clef_images_wire.py index bf6d01b2455..829d9d85ffe 100644 --- a/tests/integration/providers/test_cloudflare_clef_images_wire.py +++ b/tests/integration/providers/test_cloudflare_clef_images_wire.py @@ -321,7 +321,7 @@ def test_an_openai_decision_model_gets_system_one_images_as_input_image_parts(ga @pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions")) -def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream_call( +def test_more_images_than_clef_takes_are_refused_before_any_download_or_trimmed_under_drop_params( gateway: Gateway, route: Route ) -> None: marker: Final = uuid.uuid4().hex @@ -329,12 +329,15 @@ def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) remote: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(5)] - for model in (strict, dropping): - refused = _decide(gateway, route, model, marker, remote) - assert refused.status_code == 400, refused.text - assert "at most 4 images per request, got 5" in refused.text, refused.text + refused: Final = _decide(gateway, route, strict, marker, remote) + assert refused.status_code == 400, refused.text + assert "at most 4 images per request, got 5" in refused.text, refused.text assert wire.drain() == () four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL] + trimmed: Final = _decide(gateway, route, dropping, marker, [*four, _PNG_DATA_URL]) + assert trimmed.status_code == 200, trimmed.text + (dropped_request,) = wire.drain() + assert _upstream_body(dropped_request) == _expected_upstream_body("clef", _state(marker), four) accepted: Final = _decide(gateway, route, strict, marker, four) assert accepted.status_code == 200, accepted.text (request,) = wire.drain() @@ -500,41 +503,55 @@ def test_an_image_clef_refuses_does_not_count_toward_its_limit_when_the_deployme _MIB: Final = 1024 * 1024 +_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" + + +def _png_base64(decoded_bytes: int) -> str: + return base64.b64encode(_PNG_SIGNATURE.ljust(decoded_bytes, b"\0")).decode() + + def _png_data_url(decoded_bytes: int) -> str: - return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}" + return f"data:image/png;base64,{_png_base64(decoded_bytes)}" def _png_object(decoded_bytes: int) -> dict[str, JsonValue]: - return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} + return {"content_type": "image/png", "base64": _png_base64(decoded_bytes)} -_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], str], ...]] = ( - ("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1),), "got one of 4,194,305 bytes"), - ("one_over_4mib", "/v1/decisions", (_png_data_url(4 * _MIB + 1),), "got one of 4,194,305 bytes"), +_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], int, str], ...]] = ( + ("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1), _PNG_DATA_URL), 0, "got one of 4,194,305 bytes"), + ("one_over_4mib", "/v1/decisions", (_PNG_DATA_URL, _png_data_url(4 * _MIB + 1)), 1, "got one of 4,194,305 bytes"), ( "total_over_8mib", "/v1/systemone", - (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)), - "got 8,388,609 bytes", + (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(8)), + 2, + "got 8,388,616 bytes", ), ) @pytest.mark.parametrize( - ("label", "route", "images", "refusal"), _OVERSIZED, ids=[f"{label}{route}" for label, route, _, _ in _OVERSIZED] + ("label", "route", "images", "first_dropped", "refusal"), + _OVERSIZED, + ids=[f"{label}{route}" for label, route, _, _, _ in _OVERSIZED], ) -def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call( - gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], refusal: str +def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call_or_dropped_under_drop_params( + gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], first_dropped: int, refusal: str ) -> None: marker: Final = uuid.uuid4().hex with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) - for model in (strict, dropping): - refused = _decide(gateway, route, model, marker, images) - assert refused.status_code == 400, (label, refused.text[:500]) - assert refusal in refused.text, (label, refused.text[:500]) + refused: Final = _decide(gateway, route, strict, marker, images) + assert refused.status_code == 400, (label, refused.text[:500]) + assert refusal in refused.text, (label, refused.text[:500]) assert wire.drain() == (), label + kept: Final = (*images[:first_dropped], *images[first_dropped + 1 :]) + trimmed: Final = _decide(gateway, route, dropping, marker, images) + assert trimmed.status_code == 200, (label, trimmed.text[:500]) + (dropped_request,) = wire.drain() + assert _upstream_body(dropped_request) == _expected_upstream_body("clef", _state(marker), kept), label at_limits: Final[list[JsonValue]] = [_png_data_url(4 * _MIB), _png_object(4 * _MIB)] accepted: Final = _decide(gateway, "/v1/systemone", strict, marker, at_limits) assert accepted.status_code == 200, accepted.text[:500] @@ -848,6 +865,16 @@ def test_a_missing_remote_image_is_refused_without_a_call_to_clef(allowlisted: G @pytest.mark.timeout(_OWNED_TIMEOUT) +def test_remote_images_past_clefs_count_are_never_downloaded_under_drop_params( + allowlisted: Gateway, wire: Wire +) -> None: + marker: Final = uuid.uuid4().hex + images: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(6)] + response: Final = _decide(allowlisted, "/v1/systemone", _ALLOWLISTED_MODEL, marker, images, drop_params=True) + assert response.status_code == 200, response.text + assert sorted(method for method, _ in _routes(wire.drain())) == ["GET"] * 4 + ["POST"] + + def test_a_remote_page_that_is_not_an_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None: marker: Final = uuid.uuid4().hex response: Final = _decide( diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 0c40fe06199..b9202e2cc1d 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -162,6 +162,7 @@ _PROVIDERS: Final = ( _API_KEY, False, "perplexity/pplx-decider-v1-27b", + takes_images=True, ), _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), _Provider( @@ -172,6 +173,7 @@ _PROVIDERS: Final = ( _API_KEY, False, None, + takes_images=True, provider_reported_cost=1.5834e-5, ), _Provider( @@ -224,9 +226,10 @@ _PROVIDERS: Final = ( _SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) _TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images) _PERPLEXITY: Final = _PROVIDERS[0] +_OPENROUTER: Final = next(provider for provider in _PROVIDERS if provider.name == "openrouter") _TYPESAFE: Final = _PROVIDERS[1] _OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) -_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images) +_CLOUDFLARE: Final = next(provider for provider in _PROVIDERS if provider.name == "cloudflare") _PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"input": _INPUT}), @@ -519,6 +522,21 @@ def test_cloudflare_receives_images_from_either_format_in_its_images_field(gatew assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2 +@pytest.mark.parametrize("provider", (_PERPLEXITY, _OPENROUTER), ids=_provider_id) +def test_perplexity_and_openrouter_receive_images_from_either_format_in_their_state( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model) + assert openai_format.status_code == 200, openai_format.text + assert system_one_format.status_code == 200, system_one_format.text + image_item: Final[dict[str, JsonValue]] = {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}} + expected_body: Final = {**provider.upstream_body(), "state": [_INPUT, image_item]} + assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2 + + def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = _register(scenario, _OPENAI.upstream_reply()) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0e2813e0a0d..210ebba92f0 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1957,7 +1957,9 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_ _CLOUDFLARE_RUN_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare" _PERPLEXITY_DECISIONS_URL: Final = "https://api.perplexity.ai/v1/decisions" -_PNG_DATA_URL: Final = "data:image/png;base64,AA==" +_TEXT_ONLY_MODEL: Final = "nace/drex-v2.0" +_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone" +_PNG_DATA_URL: Final = "data:image/png;base64,iVBORw0KGgo=" _REMOTE_IMAGE_URL: Final = "https://images.example/screen.png" _SCREEN_TEXT: Final = "Is the screen cracked?" _SYSTEMONE_PREDICATE: Final[Mapping[str, object]] = MappingProxyType( @@ -2163,10 +2165,10 @@ def test_images_are_refused_before_http_when_the_provider_cannot_take_them( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: - getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) + getattr(litellm, api)(model=_TEXT_ONLY_MODEL, api_key="caller-key", **request_kwargs) assert caught.value.status_code == 400 assert not route.called @@ -2178,14 +2180,12 @@ async def test_images_are_dropped_from_the_wire_under_drop_params( api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) - await getattr(litellm, f"a{api}")( - model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs - ) + await getattr(litellm, f"a{api}")(model=_TEXT_ONLY_MODEL, api_key="caller-key", drop_params=True, **request_kwargs) assert json.loads(route.calls[0].request.content) == { - "model": "pplx-decider-v1-27b", + "model": "drex-v2.0", "state": _SCREEN_TEXT, "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } @@ -2260,13 +2260,13 @@ async def test_a_malformed_image_under_drop_params_keeps_the_well_formed_ones( model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, - images=[entry, _PNG_DATA_URL, {"content_type": "image/png", "base64": "AA=="}], + images=[entry, _PNG_DATA_URL, {"content_type": "image/png", "base64": "iVBORw0KGgo="}], drop_params=True, ) assert json.loads(route.calls[0].request.content)["images"] == [ _PNG_DATA_URL, - {"content_type": "image/png", "base64": "AA=="}, + {"content_type": "image/png", "base64": "iVBORw0KGgo="}, ] @@ -2351,7 +2351,7 @@ def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, obje def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http( api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(litellm, "drop_params", True) + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: @@ -2378,11 +2378,11 @@ async def test_dropping_the_only_input_images_is_rejected_before_http( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=r"\['input_image'\].*leaves no input") as caught: await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", + model=_TEXT_ONLY_MODEL, api_key="caller-key", input=[{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}], questions=_PREDICATE_QUESTIONS, @@ -2481,47 +2481,164 @@ async def test_dropping_the_only_malformed_input_images_is_rejected_before_http( _MIB: Final = 1024 * 1024 +_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" + + +def _png_base64(decoded_bytes: int) -> str: + return base64.b64encode(_PNG_SIGNATURE.ljust(decoded_bytes, b"\0")).decode() + + def _png_data_url(decoded_bytes: int) -> str: - return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}" + return f"data:image/png;base64,{_png_base64(decoded_bytes)}" def _png_object(decoded_bytes: int) -> Mapping[str, object]: - return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} + return {"content_type": "image/png", "base64": _png_base64(decoded_bytes)} + + +def _distinct_png_data_urls(count: int) -> tuple[str, ...]: + return tuple( + f"data:image/png;base64,{base64.b64encode(_PNG_SIGNATURE + bytes((index,))).decode()}" for index in range(count) + ) _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( ((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), ((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), ( - (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)), - "at most 8,388,608 bytes (8 MiB) of images per request, got 8,388,609 bytes", + (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(8)), + "at most 8,388,608 bytes (8 MiB) of images per request, got 8,388,616 bytes", ), ) +_OVERSIZED_IMAGE_IDS: Final = ("data_url_over_4mib", "object_over_4mib", "total_over_8mib") + + @pytest.mark.usefixtures("cloudflare_account") -@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) -@pytest.mark.parametrize( - ("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib") -) +@pytest.mark.parametrize(("images", "refusal"), _OVERSIZED_IMAGES, ids=_OVERSIZED_IMAGE_IDS) def test_images_over_cloudflares_size_limits_are_refused_before_http( - images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter + images: Sequence[object], refusal: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught: - litellm.systemone( - model="cloudflare/clef", - state=_SCREEN_TEXT, - questions=_SYSTEMONE_PREDICATE, - images=images, - drop_params=drop_params, - ) + litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images) assert caught.value.status_code == 400 assert not route.called +_SMALL_PNG: Final = _png_data_url(1) +_DROPPED_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], Sequence[object]], ...]] = ( + *((images, images[:kept]) for (images, _), kept in zip(_OVERSIZED_IMAGES, (0, 0, 2), strict=True)), + ((_png_data_url(4 * _MIB + 1), _SMALL_PNG), (_SMALL_PNG,)), + ((*(_png_data_url(4 * _MIB + 1),) * 4, _SMALL_PNG), (_SMALL_PNG,)), + ((*(_png_data_url(3 * _MIB),) * 3, _SMALL_PNG), (*(_png_data_url(3 * _MIB),) * 2, _SMALL_PNG)), +) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize( + ("images", "kept"), + _DROPPED_OVERSIZED_IMAGES, + ids=(*_OVERSIZED_IMAGE_IDS, "oversized_then_small", "four_oversized_then_small", "over_the_total_then_small"), +) +def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params( + images: Sequence[object], kept: Sequence[object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=images, + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content).get("images", []) == list(kept) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_logging_callbacks_get_only_the_images_left_after_an_oversized_one_drops( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_png_data_url(4 * _MIB + 1), _SMALL_PNG], + drop_params=True, + ) + await _drain_logging_worker() + + assert recording_logger.inputs == [json.loads(route.calls.last.request.content)] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_logging_callbacks_get_the_caller_url_of_a_remote_image_kept_after_an_oversized_one_drops( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, async_only_image_fetch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_png_data_url(4 * _MIB + 1), _REMOTE_IMAGE_URL], + drop_params=True, + ) + await _drain_logging_worker() + + wire_body: Final = json.loads(route.calls.last.request.content) + assert wire_body["images"] == [async_only_image_fetch.data_url] + assert recording_logger.inputs == [{**wire_body, "images": [_REMOTE_IMAGE_URL]}] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account", "async_only_image_fetch") +async def test_logging_callbacks_get_each_caller_url_when_two_remote_images_have_the_same_bytes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + caller_urls: Final = [_REMOTE_IMAGE_URL, f"{_REMOTE_IMAGE_URL}?copy=1"] + + await litellm.asystemone( + model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=caller_urls + ) + await _drain_logging_worker() + + assert recording_logger.inputs == [{**json.loads(route.calls.last.request.content), "images": caller_urls}] + + +@pytest.mark.usefixtures("cloudflare_account") +def test_images_past_cloudflares_limit_are_dropped_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(5) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["images"] == list(images[:4]) + + @pytest.mark.asyncio @pytest.mark.usefixtures("cloudflare_account") async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None: @@ -2583,7 +2700,7 @@ def test_remote_image_urls_with_an_uppercase_scheme_are_inlined_for_cloudflare( ] -_PNG_DATA_URL_WITH_NAME: Final = "data:image/png;name=screen.png;base64,AA==" +_PNG_DATA_URL_WITH_NAME: Final = "data:image/png;name=screen.png;base64,iVBORw0KGgo=" _GIF_DATA_URL: Final = "data:image/gif;base64,R0lGODlh" _CLOUDFLARE_TYPE_REFUSAL: Final = "cloudflare takes only image/jpeg, image/jpg, image/png, image/webp images, got " @@ -3773,6 +3890,47 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_ assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest")) +async def test_openrouter_jev_sends_box_questions_and_returns_box_answers( + model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}} + ) + + response: Final = await litellm.adecisions( + model=model, + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], + } + ], + questions=[_CRACK_BOX_QUESTION], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["questions"] == { + "crack": {"type": "box", "instructions": "the crack on the screen"} + } + assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] + + +def test_openrouter_jev_1_13_still_refuses_box_questions( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"): + litellm.decisions( + model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k" + ) + + assert not route.called + + @pytest.mark.asyncio async def test_a_deployment_that_turns_box_questions_off_is_refused_even_when_the_cost_map_allows_them( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch @@ -4819,6 +4977,906 @@ async def test_hosted_vllm_decisions_sends_input_images_to_systemone( assert async_only_image_fetch.fetched == (["https://images.example/input.png"] if remote_image else []) +_STATE_IMAGE_PROVIDERS: Final = ( + ("openrouter/cloudflare/clef", "https://openrouter.ai/api/alpha/decisions", "cloudflare/clef"), + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", "pplx-decider-v1.2"), +) +_STATE_IMAGE_PROVIDER_IDS: Final = ("openrouter", "perplexity") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +async def test_decisions_sends_input_images_in_state_order( + model: str, + url: str, + upstream_model: str, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + image: Final[str] = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA==" + expected_image: Final[str] = async_only_image_fetch.data_url if remote_image else image + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model=model, + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Listing photo:"}, + {"type": "input_image", "image_url": image}, + {"type": "input_text", "text": "Is it damaged?"}, + ], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": upstream_model, + "state": [ + "Listing photo:", + {"type": "image_url", "image_url": {"url": expected_image}}, + "Is it damaged?", + ], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].probability == 0.9 + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_systemone_moves_images_into_state( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state={"ticket": 1234}, + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + images=["data:IMAGE/JPG;name=shot.jpg;base64,A-_AB-", {"content_type": "image/webp", "base64": "B-_B"}], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": upstream_model, + "state": [ + {"ticket": 1234}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/AB+=="}}, + {"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}}, + ], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_a_list_state_keeps_its_items_ahead_of_the_images( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=["Ticket 1234", {"severity": "high"}], + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL], + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + "Ticket 1234", + {"severity": "high"}, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ] + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_line_wrapped_base64_reaches_state_on_one_line( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + image_bytes: Final = bytes(range(100)) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[{"content_type": "image/png", "base64": base64.encodebytes(image_bytes).decode()}], + api_key="k", + ) + + assert json.loads(route.calls.last.request.content)["state"][1] == { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{base64.b64encode(image_bytes).decode()}"}, + } + + +_MISSING_IMAGE_URL: Final = "https://images.example/missing.png" +_FOUND_IMAGE_URL: Final = "https://images.example/found.png" +_FOUND_PNG: Final = b"\x89PNG\r\n\x1a\n" + + +@pytest.fixture +def one_missing_remote_image(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport())) + respx_mock.get(_MISSING_IMAGE_URL).respond(status_code=404) + respx_mock.get(_FOUND_IMAGE_URL).respond(content=_FOUND_PNG, headers={"content-type": "image/png"}) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("one_missing_remote_image") +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_a_remote_image_that_fails_to_download_is_dropped_under_drop_params( + sync: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + request_kwargs: Final = { + "model": "perplexity/pplx-decider-v1.2", + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": [_MISSING_IMAGE_URL, _FOUND_IMAGE_URL], + "api_key": "k", + "drop_params": True, + } + + if sync: + litellm.systemone(**request_kwargs) + else: + await litellm.asystemone(**request_kwargs) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64.b64encode(_FOUND_PNG).decode()}"}}, + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("one_missing_remote_image") +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_remote_images_past_the_count_are_never_downloaded_under_drop_params( + sync: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + urls: Final = tuple(f"https://images.example/{index}.png" for index in range(10)) + downloads: Final = tuple( + respx_mock.get(url).respond(content=_FOUND_PNG, headers={"content-type": "image/png"}) for url in urls + ) + request_kwargs: Final = { + "model": "perplexity/pplx-decider-v1.2", + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": list(urls), + "api_key": "k", + "drop_params": True, + } + + if sync: + litellm.systemone(**request_kwargs) + else: + await litellm.asystemone(**request_kwargs) + + assert [download.call_count for download in downloads] == [1] * 8 + [0] * 2 + assert len(json.loads(route.calls.last.request.content)["state"]) == 9 + + +_UNDROPPABLE_DOWNLOAD_FAILURES: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + ("asystemone", {"state": _SCREEN_TEXT, "questions": _SYSTEMONE_PREDICATE, "images": [_MISSING_IMAGE_URL]}), + ( + "adecisions", + { + "input": [{"role": "user", "content": [{"type": "input_image", "image_url": _MISSING_IMAGE_URL}]}], + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "drop_params": True, + }, + ), +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("one_missing_remote_image") +@pytest.mark.parametrize( + ("api", "request_kwargs"), + _UNDROPPABLE_DOWNLOAD_FAILURES, + ids=("without_drop_params", "only_input_under_drop_params"), +) +async def test_a_remote_image_that_fails_to_download_is_refused_before_http_when_it_cannot_be_dropped( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="Status code: 404"): + await getattr(litellm, api)(model="perplexity/pplx-decider-v1.2", api_key="k", **request_kwargs) + + assert route.call_count == 0 + + +_NOT_AN_IMAGE_BASE64: Final = base64.b64encode(b"not an image").decode() +_PERPLEXITY_IMAGE_TYPE_MODELS: Final = ( + ("perplexity/pplx-decider-v1.2", _PERPLEXITY_DECISIONS_URL), + ("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions"), +) +_PERPLEXITY_IMAGE_TYPE_MODEL_IDS: Final = ("perplexity", "openrouter-perplexity") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +@pytest.mark.parametrize(("model", "url"), _PERPLEXITY_IMAGE_TYPE_MODELS, ids=_PERPLEXITY_IMAGE_TYPE_MODEL_IDS) +async def test_image_bytes_that_are_not_an_image_the_model_takes_are_dropped_under_drop_params( + sync: bool, model: str, url: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + gif: Final = f"data:image/png;base64,{base64.b64encode(b'GIF89a').decode()}" + request_kwargs: Final = { + "model": model, + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": [ + {"content_type": "image/png", "base64": _NOT_AN_IMAGE_BASE64}, + {"content_type": "image/png", "base64": "\u00e9" * 8}, + gif, + _PNG_DATA_URL, + ], + "api_key": "k", + "drop_params": True, + } + + if sync: + litellm.systemone(**request_kwargs) + else: + await litellm.asystemone(**request_kwargs) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ] + + +@pytest.mark.parametrize(("model", "url"), _PERPLEXITY_IMAGE_TYPE_MODELS, ids=_PERPLEXITY_IMAGE_TYPE_MODEL_IDS) +def test_image_bytes_that_are_not_an_image_still_reach_the_model_without_drop_params( + model: str, url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + not_an_image: Final = f"data:image/png;base64,{_NOT_AN_IMAGE_BASE64}" + + litellm.systemone(model=model, state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[not_an_image], api_key="k") + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": not_an_image}}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "url"), _PERPLEXITY_IMAGE_TYPE_MODELS, ids=_PERPLEXITY_IMAGE_TYPE_MODEL_IDS) +async def test_image_bytes_that_are_not_an_image_are_refused_before_http_when_nothing_else_is_left( + model: str, url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"image bytes, got 1 image\(s\) that are none of these"): + await litellm.adecisions( + model=model, + api_key="k", + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": f"data:image/png;base64,{_NOT_AN_IMAGE_BASE64}"}], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + drop_params=True, + ) + + assert route.call_count == 0 + + +_UNDECODABLE_PNG_DATA_URL: Final = f"data:image/png;base64,{base64.b64encode(_PNG_SIGNATURE + b'IHDR').decode()}" +_UNDECODABLE_PNG_REQUEST: Final[Mapping[str, object]] = MappingProxyType( + { + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": [_UNDECODABLE_PNG_DATA_URL, _PNG_DATA_URL], + "api_key": "k", + "drop_params": True, + } +) + + +def _refused_then_answered(status: int) -> tuple[httpx.Response, httpx.Response]: + return httpx.Response(status, json={"error": {"message": "refused"}}), httpx.Response(200, json=_RESPONSE) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +@pytest.mark.parametrize("status", (400, 408, 413, 422)) +@pytest.mark.parametrize(("model", "url"), _PERPLEXITY_IMAGE_TYPE_MODELS, ids=_PERPLEXITY_IMAGE_TYPE_MODEL_IDS) +async def test_a_request_whose_images_the_model_refuses_is_sent_again_without_them_under_drop_params( + sync: bool, status: int, model: str, url: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).mock(side_effect=_refused_then_answered(status)) + + if sync: + litellm.systemone(model=model, **_UNDECODABLE_PNG_REQUEST) + else: + await litellm.asystemone(model=model, **_UNDECODABLE_PNG_REQUEST) + + assert [json.loads(call.request.content)["state"] for call in route.calls] == [ + [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": _UNDECODABLE_PNG_DATA_URL}}, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ], + _SCREEN_TEXT, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_a_request_whose_images_time_out_is_sent_again_without_them_under_drop_params( + sync: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).mock( + side_effect=(httpx.ReadTimeout("too slow"), httpx.Response(200, json=_RESPONSE)) + ) + + if sync: + litellm.systemone(model="perplexity/pplx-decider-v1.2", **_UNDECODABLE_PNG_REQUEST) + else: + await litellm.asystemone(model="perplexity/pplx-decider-v1.2", **_UNDECODABLE_PNG_REQUEST) + + assert route.call_count == 2 + assert json.loads(route.calls.last.request.content)["state"] == _SCREEN_TEXT + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_a_refusal_from_a_model_that_always_took_images_is_not_sent_again_under_drop_params( + sync: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").mock(side_effect=_refused_then_answered(422)) + request_kwargs: Final = { + "model": "cloudflare/clef", + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": [_PNG_DATA_URL], + "drop_params": True, + } + + if sync: + with pytest.raises(litellm.BadRequestError): + litellm.systemone(**request_kwargs) + else: + with pytest.raises(litellm.BadRequestError): + await litellm.asystemone(**request_kwargs) + + assert route.call_count == 1 + + +_NOT_AN_IMAGE_REQUEST: Final[Mapping[str, object]] = MappingProxyType( + { + "questions": _SYSTEMONE_PREDICATE, + "images": [{"content_type": "image/png", "base64": _NOT_AN_IMAGE_BASE64}], + "drop_params": True, + } +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize( + "state", + ( + "", + [], + {}, + " ", + [""], + {"a": ""}, + {"a": [" "]}, + [None], + {"a": None}, + [{"type": "input_text", "text": ""}], + [{"type": "text", "text": " "}], + ), + ids=( + "string", + "list", + "object", + "blank", + "list-of-empty", + "object-of-empty", + "nested-blank", + "list-of-null", + "object-of-null", + "input-text-part-with-empty-text", + "text-part-with-blank-text", + ), +) +async def test_an_empty_state_with_only_a_dropped_image_is_refused_when_the_model_always_took_images( + state: object, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError): + await litellm.asystemone(model="cloudflare/clef", state=state, **_NOT_AN_IMAGE_REQUEST) + + assert route.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("state", ("a", ["a"], {"a": "b"}, {"a": 0}, [False], {"type": "refund"}), ids=str) +async def test_a_state_with_something_in_it_still_reaches_the_model_after_its_only_image_is_dropped( + state: object, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.asystemone(model="cloudflare/clef", state=state, **_NOT_AN_IMAGE_REQUEST) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["state"] == state + assert sent.get("images", []) == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("state", ("", [], {}, " ", [""], {"a": ""}, [None]), ids=str) +async def test_a_blank_state_still_reaches_the_model_when_its_only_image_drops_for_its_declared_type( + state: object, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.asystemone( + model="cloudflare/clef", + state=state, + questions=_SYSTEMONE_PREDICATE, + images=[{"content_type": "image/gif", "base64": "R0lGODlh"}], + drop_params=True, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["state"] == state + assert sent.get("images", []) == [] + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("state", ("", [], " ", [None]), ids=str) +def test_a_blank_state_still_reaches_the_model_when_its_only_download_is_a_type_the_model_refuses( + state: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + respx_mock.get(_REMOTE_IMAGE_URL).respond(content=b"GIF89a", headers={"content-type": "image/gif"}) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=state, + questions=_SYSTEMONE_PREDICATE, + images=[_REMOTE_IMAGE_URL], + drop_params=True, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["state"] == state + assert sent.get("images", []) == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_a_blank_state_is_refused_when_its_downloads_drop_for_their_type_after_the_count_drop( + sync: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + respx_mock.get(url__regex=r"^https://images\.example/screen-\d\.gif$").respond( + content=b"GIF89a", headers={"content-type": "image/gif"} + ) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + request_kwargs: Final = { + "model": "cloudflare/clef", + "state": "", + "questions": _SYSTEMONE_PREDICATE, + "images": [f"https://images.example/screen-{index}.gif" for index in range(5)], + "drop_params": True, + } + + if sync: + with pytest.raises(litellm.BadRequestError): + litellm.systemone(**request_kwargs) + else: + with pytest.raises(litellm.BadRequestError): + await litellm.asystemone(**request_kwargs) + + assert not route.called + + +@pytest.mark.usefixtures("cloudflare_account") +def test_a_blank_state_is_refused_when_a_download_type_drop_comes_with_a_download_size_drop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + gif_url: Final = "https://images.example/screen.gif" + respx_mock.get(gif_url).respond(content=b"GIF89a", headers={"content-type": "image/gif"}) + respx_mock.get(_REMOTE_IMAGE_URL).respond(content=bytes(4 * _MIB + 1), headers={"content-type": "image/png"}) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError): + litellm.systemone( + model="cloudflare/clef", + state="", + questions=_SYSTEMONE_PREDICATE, + images=[gif_url, _REMOTE_IMAGE_URL], + drop_params=True, + ) + + assert not route.called + + +_NOT_AN_IMAGE_DATA_URL: Final = f"data:image/png;base64,{_NOT_AN_IMAGE_BASE64}" +_DECISIONS_PREDICATE: Final[list[dict[str, str]]] = [{"type": "predicate", "name": "red", "instructions": "Is it red?"}] + + +def _decisions_input_with_text_and_dropped_image(text: str) -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": text}, + {"type": "input_image", "image_url": _NOT_AN_IMAGE_DATA_URL}, + ], + } + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("text", ("", " "), ids=("empty", "blank")) +async def test_decisions_input_with_no_text_and_only_a_dropped_image_is_refused( + text: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError): + await litellm.adecisions( + model="cloudflare/clef", + input=_decisions_input_with_text_and_dropped_image(text), + questions=_DECISIONS_PREDICATE, + drop_params=True, + ) + + assert route.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_decisions_input_with_no_text_still_reaches_the_model_when_its_image_drops_for_its_declared_type( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": ""}, + {"type": "input_image", "image_url": "data:image/gif;base64,R0lGODlh"}, + ], + } + ], + questions=_DECISIONS_PREDICATE, + drop_params=True, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["state"] == "" + assert sent.get("images", []) == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_decisions_input_with_text_still_reaches_the_model_after_its_only_image_is_dropped( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + input=_decisions_input_with_text_and_dropped_image("Product photo"), + questions=_DECISIONS_PREDICATE, + drop_params=True, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["state"] == "Product photo" + assert sent.get("images", []) == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize( + "clips", + ({"audio": ["data:audio/wav;base64,YQ=="]}, {"videos": ["data:video/mp4;base64,YQ=="]}), + ids=("audio", "videos"), +) +async def test_an_empty_state_with_only_a_dropped_image_still_reaches_the_model_with_its_clips( + clips: Mapping[str, list[str]], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef-omni").respond(json=_RESPONSE) + + await litellm.asystemone(model="cloudflare/clef-omni", state="", **_NOT_AN_IMAGE_REQUEST, **clips) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent.get("images", []) == [] + assert {name: sent[name] for name in clips} == clips + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_an_empty_state_keeps_the_images_that_survive_a_drop(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.asystemone( + model="cloudflare/clef", + state="", + questions=_SYSTEMONE_PREDICATE, + images=[{"content_type": "image/png", "base64": _NOT_AN_IMAGE_BASE64}, _PNG_DATA_URL], + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["images"] == [_PNG_DATA_URL] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("state", ("", [], {}), ids=("string", "list", "object")) +async def test_an_empty_state_with_only_a_dropped_image_still_reaches_a_model_that_took_no_images_on_main( + state: object, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + await litellm.asystemone(model="perplexity/pplx-decider-v1.2", state=state, api_key="k", **_NOT_AN_IMAGE_REQUEST) + + assert json.loads(route.calls.last.request.content)["state"] == state + + +@pytest.mark.asyncio +async def test_logging_callbacks_get_the_request_sent_again_without_images( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).mock(side_effect=_refused_then_answered(400)) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone(model="perplexity/pplx-decider-v1.2", **_UNDECODABLE_PNG_REQUEST) + await _drain_logging_worker() + + assert recording_logger.inputs == [json.loads(route.calls.last.request.content)] + + +_REFUSALS_SENT_ONCE: Final[tuple[tuple[str, Mapping[str, object], int, type[Exception]], ...]] = ( + ("asystemone", {**_UNDECODABLE_PNG_REQUEST, "drop_params": False}, 400, litellm.BadRequestError), + ("asystemone", _UNDECODABLE_PNG_REQUEST, 401, litellm.AuthenticationError), + ( + "adecisions", + { + "input": [{"role": "user", "content": [{"type": "input_image", "image_url": _UNDECODABLE_PNG_DATA_URL}]}], + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "api_key": "k", + "drop_params": True, + }, + 400, + litellm.BadRequestError, + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api", "request_kwargs", "status", "error_class"), + _REFUSALS_SENT_ONCE, + ids=("without_drop_params", "not_a_refusal_of_the_content", "no_input_left_without_images"), +) +async def test_a_refusal_reaches_the_caller_after_one_call_when_the_images_cannot_be_dropped( + api: str, + request_kwargs: Mapping[str, object], + status: int, + error_class: type[Exception], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).mock(side_effect=_refused_then_answered(status)) + + with pytest.raises(error_class): + await getattr(litellm, api)(model="perplexity/pplx-decider-v1.2", **request_kwargs) + + assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +async def test_logging_callbacks_get_the_state_that_was_sent_with_its_images( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model=model, state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_PNG_DATA_URL], api_key="k" + ) + await _drain_logging_worker() + + assert recording_logger.inputs == [json.loads(route.calls.last.request.content)] + + +_STATE_IMAGE_LIMITS: Final = ( + ("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8), + ("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4), + ("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions", 8), +) +_STATE_IMAGE_LIMIT_IDS: Final = ("perplexity", "openrouter-clef", "openrouter-perplexity") + + +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) +def test_more_images_than_the_model_takes_are_refused_before_http( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * max_images, + api_key="k", + ) + with pytest.raises( + litellm.BadRequestError, + match=re.escape(f"takes at most {max_images} images per request, got {max_images + 1}"), + ): + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL] * (max_images + 1), + api_key="k", + ) + + assert route.call_count == 1 + + +@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS) +def test_images_past_the_models_limit_are_dropped_under_drop_params( + model: str, url: str, max_images: int, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + images: Final = _distinct_png_data_urls(max_images + 2) + + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=list(images), + api_key="k", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + *({"type": "image_url", "image_url": {"url": image}} for image in images[:max_images]), + ] + + +def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8) + + with pytest.raises(litellm.BadRequestError, match=re.escape("openrouter takes images of at most 4,194,304 bytes")): + litellm.systemone( + model="openrouter/cloudflare/clef-flash", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[over_four_mib], + api_key="k", + ) + + assert not route.called + + +@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS) +def test_a_gif_to_a_state_image_provider_is_rejected_before_http( + model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + with pytest.raises( + litellm.BadRequestError, match=re.escape("takes only image/jpeg, image/jpg, image/png, image/webp images") + ) as caught: + litellm.systemone( + model=model, state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_GIF_DATA_URL], api_key="k" + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest")) +def test_openrouter_jev_rejects_webp_before_http( + model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=re.escape("got image/webp")): + litellm.systemone( + model=model, + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=["data:image/webp;base64,BB=="], + api_key="k", + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_images_to_an_openrouter_upstream_without_image_rules_are_refused_or_dropped_as_before( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + request: Final[Mapping[str, object]] = MappingProxyType( + { + "model": "openrouter/upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + "images": [_SMALL_PNG], + "api_key": "caller-key", + } + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['images'\]"): + await litellm.asystemone(**request) + await litellm.asystemone(**request, drop_params=True) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == { + "model": "upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + } + + +def test_openrouter_jev_drops_webp_and_keeps_png_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + + litellm.systemone( + model="openrouter/typesafe/jev-1.14", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=["data:image/webp;base64,BB==", _PNG_DATA_URL], + api_key="k", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["state"] == [ + _SCREEN_TEXT, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + ] + + def test_hosted_vllm_systemone_forwards_images( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 2ef202762ff..7e5446fe121 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -1,5 +1,7 @@ import json +import sys from collections.abc import Mapping +from functools import reduce from typing import Final import httpx @@ -13,6 +15,7 @@ from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsInputLimits, DecisionsResponseValidationError, + input_has_no_text, ir_to_systemone_request, ir_to_systemone_response, parse_systemone_response, @@ -26,6 +29,7 @@ from litellm.types.decisions import ( DecisionsIRRefusal, DecisionsIRRequest, DecisionsIRScoreAnswer, + DecisionsIRState, DecisionsRequestBody, OpenAIDecisionRequestBody, ) @@ -517,3 +521,13 @@ def test_non_json_provider_response_is_an_api_error(raw_content: bytes, provider with pytest.raises(BaseLLMException, match="unexpected response") as error: config.transform_decisions_response("drex-v2.0", provider, httpx.Response(200, content=raw_content), request) assert error.value.status_code == 500 + + +def _nested_once(inner: object, _: int) -> object: + return [inner] + + +def test_input_has_no_text_counts_a_state_nested_past_the_stack_as_no_text() -> None: + deepest_state: Final = reduce(_nested_once, range(sys.getrecursionlimit() * 2), "Product photo") + + assert input_has_no_text(DecisionsIRState(state=deepest_state, images=())) is True diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..bf7c56506e9 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -937,7 +937,7 @@ def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_p _IMAGE_TEXT: Final = "Is the screen cracked?" -_PNG_DATA_URL: Final = "data:image/png;base64,AA==" +_PNG_DATA_URL: Final = "data:image/png;base64,iVBORw0KGgo=" _IMAGE_ENDPOINTS: Final = ("/v1/decisions", "/v1/systemone") _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = ( { @@ -962,6 +962,8 @@ _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = ( ) _IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True)) _IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format") +_TEXT_ONLY_DEPLOYMENT: Final[Mapping[str, object]] = {"model": "nace/drex-v2.0", "api_key": "test-key"} +_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone" def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None: @@ -1017,7 +1019,8 @@ def test_images_to_a_text_only_deployment_are_refused_without_drop_params( param: str, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + _route_decider_to(monkeypatch, _TEXT_ONLY_DEPLOYMENT) + upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) response: Final = client.post(endpoint, json=request_body) @@ -1035,16 +1038,14 @@ def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params( request_body: Mapping[str, object], ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - _route_decider_to( - monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True} - ) - upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + _route_decider_to(monkeypatch, {**_TEXT_ONLY_DEPLOYMENT, "drop_params": True}) + upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE) response: Final = client.post(endpoint, json=request_body) assert response.status_code == 200, response.text assert json.loads(upstream.calls[0].request.content) == { - "model": "pplx-decider-v1-27b", + "model": "drex-v2.0", "state": _IMAGE_TEXT, "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } @@ -1104,7 +1105,12 @@ def test_malformed_assets_for_a_deployment_without_asset_support_are_dropped_onl response: Final = client.post( "/v1/systemone", - json={**_IMAGE_REQUEST_BODIES[1], "assets": {"photo-1": {"type": "image", "part": "file1"}}}, + json={ + "model": "decider", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "assets": {"photo-1": {"type": "image", "part": "file1"}}, + }, ) if not deployment_drops_params: