diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index d2be14e5529..ef78cb68323 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,5 +1,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace +from functools import partial +from types import MappingProxyType from typing import Final, TypeAlias import httpx @@ -9,19 +11,33 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_urls_to_base64, + convert_url_to_base64, +) from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, + image_media_type, + image_param, + input_images, ir_to_systemone_response, + remote_image_urls, systemone_request_to_ir, + with_images_where, + with_inlined_images, ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir from litellm.types.decisions import ( DecisionQuestion, DecisionsAudio, + DecisionsBase64Image, + DecisionsImage, + DecisionsIRMessages, DecisionsIRRequest, DecisionsIRResponse, + DecisionsIRState, DecisionsJSON, DecisionsRequestBody, DecisionsResponse, @@ -37,6 +53,7 @@ from litellm.utils import ProviderConfigManager SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] +DecisionsImages: TypeAlias = Sequence[str | DecisionsBase64Image | Mapping[str, object]] DecisionsAudioClips: TypeAlias = Sequence[DecisionsAudio | Mapping[str, object]] DecisionsVideoClips: TypeAlias = Sequence[DecisionsVideo | Mapping[str, object]] DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody @@ -46,19 +63,62 @@ _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAda _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( str | None, config=ConfigDict(title="safety_identifier") ) +_IMAGES_ADAPTER: Final[TypeAdapter[tuple[DecisionsImage, ...] | None]] = TypeAdapter( + tuple[DecisionsImage, ...] | None, config=ConfigDict(title="images") +) _HANDLER: Final = BaseLLMHTTPHandler() +_BASE64_MARKER: Final = ";base64," + + +_IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images")) +_IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) + + +def _valid_image(image: object) -> DecisionsImage | None: + try: + return _IMAGE_ADAPTER.validate_python(image) + except ValidationError: + return None + + +def _image_entries(images: object) -> tuple[object, ...]: + try: + return _IMAGE_ENTRIES_ADAPTER.validate_python(images) + except ValidationError: + return () + + +def _is_well_formed_image(image: DecisionsImage) -> bool: + return _valid_image(image) is not None + + +def _no_image(_image: DecisionsImage) -> bool: + return False + + +def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]: + if not drops_params: + return _IMAGES_ADAPTER.validate_python(images) or () + return tuple(image for image in map(_valid_image, _image_entries(images)) if image is not None) @dataclass(frozen=True, slots=True) class SystemOneRequestFields: state: DecisionsJSON | None questions: SystemOneQuestions | None + images: object = None audio: DecisionsAudioClips | None = None videos: DecisionsVideoClips | None = None - def validate(self, safety_identifier: str | None) -> DecisionsRequestBody: + def validate(self, *, safety_identifier: str | None, drops_params: bool) -> DecisionsRequestBody: return _SYSTEMONE_REQUEST_ADAPTER.validate_python( - {"state": self.state, "questions": self.questions, "audio": self.audio, "videos": self.videos} + { + "state": self.state, + "questions": self.questions, + "images": _request_images(self.images, drops_params) or None, + "audio": self.audio, + "videos": self.videos, + } ) @@ -69,7 +129,7 @@ class OpenAIDecisionRequestFields: audio: DecisionsAudioClips | None = None videos: DecisionsVideoClips | None = None - def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody: + def validate(self, *, safety_identifier: str | None, drops_params: bool) -> OpenAIDecisionRequestBody: return _OPENAI_REQUEST_ADAPTER.validate_python( { "input": self.input, @@ -92,7 +152,6 @@ class DecisionsCall: provider_config: BaseDecisionsConfig request: DecisionsRequestFormat ir_request: DecisionsIRRequest - body: Mapping[str, object] = field(repr=False) api_base: str api_key: str | None = field(repr=False) logging_obj: LiteLLMLoggingObj | None @@ -169,18 +228,61 @@ class _UnsupportedParams: names: tuple[str, ...] -def _unsupported_param_names( +@dataclass(frozen=True, slots=True) +class _NoInputLeftAfterDrop: + names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _MalformedImages: + count: int + + +def _unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str ) -> tuple[str, ...]: - checks: Final = ( - ( - "safety_identifier", - ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier, - ), - ("audio", bool(ir_request.audio) and not provider_config.supports_audio_input(model, custom_llm_provider)), - ("videos", bool(ir_request.videos) and not provider_config.supports_video_input(model, custom_llm_provider)), + refuses_safety_identifier: Final = ( + ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) - return tuple(name for name, unsupported in checks if unsupported) + refused_image_param: Final = None if provider_config.supports_images else image_param(ir_request.input) + refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input( + model, custom_llm_provider + ) + refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input( + model, custom_llm_provider + ) + return ( + *(("safety_identifier",) if refuses_safety_identifier else ()), + *(() if refused_image_param is None else (refused_image_param,)), + *(("audio",) if refuses_audio else ()), + *(("videos",) if refuses_videos else ()), + ) + + +def _without_unsupported_params( + ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig +) -> DecisionsIRRequest: + return replace( + ir_request, + input=ir_request.input if provider_config.supports_images else with_images_where(ir_request.input, _no_image), + safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, + ) + + +def _leaves_no_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> bool: + return isinstance(decision_input, DecisionsIRMessages) and not decision_input.messages + + +def _without_malformed_images( + ir_request: DecisionsIRRequest, drops_params: bool +) -> DecisionsIRRequest | _MalformedImages: + malformed: Final = sum(1 for image in input_images(ir_request.input) if not _is_well_formed_image(image)) + if not malformed: + return ir_request + well_formed: Final = replace(ir_request, input=with_images_where(ir_request.input, _is_well_formed_image)) + if not drops_params or _leaves_no_input(well_formed.input): + return _MalformedImages(count=malformed) + return well_formed def _provider_ir_request( @@ -190,12 +292,20 @@ def _provider_ir_request( provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, -) -> DecisionsIRRequest | _UnsupportedParams: - ir_request: Final = _ir_request(request, safety_identifier) - unsupported: Final = _unsupported_param_names(ir_request, provider_config, model, custom_llm_provider) - if unsupported: - return _UnsupportedParams(unsupported) - return ir_request + kwargs: Mapping[str, object], +) -> DecisionsIRRequest | _UnsupportedParams | _NoInputLeftAfterDrop | _MalformedImages: + ir_request: Final = _without_malformed_images(_ir_request(request, safety_identifier), _drops_params(kwargs)) + if isinstance(ir_request, _MalformedImages): + return ir_request + unsupported: Final = _unsupported_params(ir_request, provider_config, model, custom_llm_provider) + if not unsupported: + return ir_request + if not _drops_params(kwargs): + return _UnsupportedParams(names=unsupported) + supported: Final = _without_unsupported_params(ir_request, provider_config) + if _leaves_no_input(supported.input): + return _NoInputLeftAfterDrop(names=unsupported) + return supported def prepare_call( @@ -231,7 +341,7 @@ def prepare_call( request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) request: Final = _fields_without_unsupported_media( fields, provider_config, canonical_model, provider, kwargs - ).validate(request_safety_identifier) + ).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -260,6 +370,7 @@ def prepare_call( provider_config=provider_config, model=canonical_model, custom_llm_provider=provider, + kwargs=kwargs, ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( @@ -271,16 +382,24 @@ def prepare_call( model=model, llm_provider=provider, ) - body: Final = provider_config.transform_decisions_request( - model=canonical_model, request=ir_request, custom_llm_provider=provider - ) - if isinstance(body, UnsupportedDecisionsRequest): + if isinstance(ir_request, _MalformedImages): raise litellm.BadRequestError( - message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", + message=( + f"Invalid Decisions request: {ir_request.count} input_image part(s) have an image_url that is " + "neither a data:image/...;base64, URL nor an http(s) URL" + ), + model=model, + llm_provider=provider, + ) + if isinstance(ir_request, _NoInputLeftAfterDrop): + raise litellm.BadRequestError( + message=( + f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, " + "and dropping them leaves no input to decide on" + ), model=model, llm_provider=provider, ) - logging_obj: Final = kwargs.get("litellm_logging_obj") if isinstance(logging_obj, LiteLLMLoggingObj): logging_obj.update_from_kwargs( @@ -299,7 +418,6 @@ def prepare_call( provider_config=provider_config, request=request, ir_request=ir_request, - body=body, api_base=resolved_api_base, api_key=resolved_api_key, logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, @@ -309,6 +427,122 @@ def prepare_call( ) +def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]: + if not call.provider_config.inlines_remote_images: + return () + return remote_image_urls(call.ir_request.input) + + +def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall: + return replace( + call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls)) + ) + + +def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall: + remote_urls: Final = _remote_image_urls(call) + if not remote_urls: + return call + return _with_data_urls( + call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls}) + ) + + +async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall: + remote_urls: Final = _remote_image_urls(call) + if not remote_urls: + return call + data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout) + return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))) + + +def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool: + media_type: Final = image_media_type(image) + return media_type is None or media_type in media_types + + +def _checked_image_types(call: DecisionsCall) -> DecisionsCall: + media_types: Final = call.provider_config.image_media_types + if media_types is None: + return call + found: Final = (image_media_type(image) for image in input_images(call.ir_request.input)) + unsupported: Final = tuple( + dict.fromkeys(type_ for type_ in found if type_ is not None and type_ not in media_types) + ) + if not unsupported: + return call + supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)) + if _drops_params(call.litellm_params) and not _leaves_no_input(supported): + return replace(call, ir_request=replace(call.ir_request, input=supported)) + raise litellm.BadRequestError( + message=( + f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, " + f"got {', '.join(unsupported)}" + ), + model=call.requested_model, + llm_provider=call.custom_llm_provider, + ) + + +def _checked_image_count(call: DecisionsCall) -> DecisionsCall: + max_images: Final = call.provider_config.max_images + image_count: Final = len(input_images(call.ir_request.input)) + if max_images is None or image_count <= max_images: + return call + raise litellm.BadRequestError( + message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}", + model=call.requested_model, + llm_provider=call.custom_llm_provider, + ) + + +def _decoded_image_bytes(image: DecisionsImage) -> int: + if isinstance(image, DecisionsBase64Image): + return len(image.base64.rstrip("=")) * 3 // 4 + marker_at: Final = image.lower().find(_BASE64_MARKER) + return 0 if marker_at < 0 else len(image[marker_at + len(_BASE64_MARKER) :].rstrip("=")) * 3 // 4 + + +def _byte_limit(limit: int) -> str: + return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)" + + +def _image_size_error(call: DecisionsCall) -> str | None: + sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input)) + largest: Final = max(sizes, default=0) + per_image: Final = call.provider_config.max_image_bytes + total: Final = call.provider_config.max_total_image_bytes + if per_image is not None and largest > per_image: + return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes" + if total is not None and sum(sizes) > total: + return f"takes at most {_byte_limit(total)} of images per request, got {sum(sizes):,} bytes" + return None + + +def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall: + error: Final = _image_size_error(call) + if error is None: + return call + raise litellm.BadRequestError( + message=f"{call.custom_llm_provider} {error}", + model=call.requested_model, + llm_provider=call.custom_llm_provider, + ) + + +def _request_body(call: DecisionsCall) -> Mapping[str, object]: + body: Final = call.provider_config.transform_decisions_request( + model=call.model, request=call.ir_request, custom_llm_provider=call.custom_llm_provider + ) + if isinstance(body, UnsupportedDecisionsRequest): + raise litellm.BadRequestError( + message=f"Decisions provider '{call.custom_llm_provider}' cannot serve this request: {body.reason}", + model=call.requested_model, + llm_provider=call.custom_llm_provider, + ) + return body + + def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: formatted: Final = _formatted_response(response, call) provider_cost: Final = call.provider_config.provider_reported_cost(response) @@ -351,43 +585,51 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + supported: Final = _checked_image_count(_checked_image_types(call)) + inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported))) + body: Final = _request_body(inlined) try: response: Final = await _HANDLER.adecisions( - model=call.model, - custom_llm_provider=call.custom_llm_provider, - logging_obj=call.logging_obj, - provider_config=call.provider_config, - request=call.ir_request, - body=call.body, - api_base=call.api_base, - api_key=call.api_key, - headers=call.headers, - timeout=call.timeout, - litellm_params=call.litellm_params, + model=inlined.model, + custom_llm_provider=inlined.custom_llm_provider, + logging_obj=inlined.logging_obj, + provider_config=inlined.provider_config, + request=inlined.ir_request, + body=body, + logged_body=body if inlined is supported else _request_body(supported), + api_base=inlined.api_base, + api_key=inlined.api_key, + headers=inlined.headers, + timeout=inlined.timeout, + litellm_params=inlined.litellm_params, ) except Exception as error: - raise _map_upstream_exception(error, call) from error - return _format_response(response, call) + raise _map_upstream_exception(error, inlined) from error + return _format_response(response, inlined) def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + supported: Final = _checked_image_count(_checked_image_types(call)) + inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported))) + body: Final = _request_body(inlined) try: response: Final = _HANDLER.decisions( - model=call.model, - custom_llm_provider=call.custom_llm_provider, - logging_obj=call.logging_obj, - provider_config=call.provider_config, - request=call.ir_request, - body=call.body, - api_base=call.api_base, - api_key=call.api_key, - headers=call.headers, - timeout=call.timeout, - litellm_params=call.litellm_params, + model=inlined.model, + custom_llm_provider=inlined.custom_llm_provider, + logging_obj=inlined.logging_obj, + provider_config=inlined.provider_config, + request=inlined.ir_request, + body=body, + logged_body=body if inlined is supported else _request_body(supported), + api_base=inlined.api_base, + api_key=inlined.api_key, + headers=inlined.headers, + timeout=inlined.timeout, + litellm_params=inlined.litellm_params, ) except Exception as error: - raise _map_upstream_exception(error, call) from error - return _format_response(response, call) + raise _map_upstream_exception(error, inlined) from error + return _format_response(response, inlined) def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None: @@ -398,3 +640,16 @@ def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: model=model, llm_provider="", ) + + +def reject_images_next_to_input(*, model: str, kwargs: Mapping[str, object]) -> None: + if not kwargs.get("images") or _drops_params(kwargs): + return + raise litellm.BadRequestError( + message=( + "images belongs to the System One format; with input (OpenAI format), " + "send images as input_image content parts" + ), + model=model, + llm_provider="", + ) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 921243612c7..6b278f03030 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -16,6 +16,7 @@ from litellm.decisions.call import ( SystemOneRequestFields, asend, prepare_call, + reject_images_next_to_input, send, ) from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse @@ -26,9 +27,11 @@ _SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = Ty _OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( OpenAIDecisionQuestions | None ) +_SYSTEMONE_FIELDS: Final = frozenset({"state", "images"}) def _request_fields( + model: str, decision_input: OpenAIDecisionInput | None, questions: OpenAIDecisionQuestions | SystemOneQuestions | None, audio: DecisionsAudioClips | None, @@ -36,6 +39,7 @@ def _request_fields( kwargs: Mapping[str, object], ) -> RequestFields: if "state" not in kwargs: + reject_images_next_to_input(model=model, kwargs=kwargs) return OpenAIDecisionRequestFields( input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions), @@ -50,6 +54,7 @@ def _request_fields( return SystemOneRequestFields( state=_STATE_ADAPTER.validate_python(kwargs["state"]), questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions), + images=kwargs.get("images"), audio=audio, videos=videos, ) @@ -72,14 +77,14 @@ def _prepare_decisions_call( ) -> DecisionsCall: return prepare_call( model=model, - fields=_request_fields(decision_input, questions, audio, videos, kwargs), + fields=_request_fields(model, decision_input, questions, audio, videos, kwargs), safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, - kwargs={key: value for key, value in kwargs.items() if key != "state"}, + kwargs={key: value for key, value in kwargs.items() if key not in _SYSTEMONE_FIELDS}, ) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 7924bb9bf4c..4cf76e5bd5e 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -4,12 +4,14 @@ Helper functions to handle images passed in messages import asyncio import base64 +import time from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass from itertools import chain from types import MappingProxyType from typing import Final +import httpx from httpx import Response import litellm @@ -22,6 +24,7 @@ from litellm.types.llms.openai import AllMessageValues MAX_IMGS_IN_MEMORY: Final = 10 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20 +_FETCH_ATTEMPTS: Final = 3 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY) @@ -73,6 +76,13 @@ def _process_image_response(response: Response, url: str) -> str: return result +def _attempt_timeouts(timeout: float | httpx.Timeout | None) -> Iterable[float | httpx.Timeout | None]: + if timeout is None or isinstance(timeout, httpx.Timeout): + return (timeout,) * _FETCH_ATTEMPTS + deadline: Final = time.monotonic() + timeout + return (remaining for _ in range(_FETCH_ATTEMPTS) if (remaining := deadline - time.monotonic()) > 0) + + def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchError": verbose_logger.warning("Image fetch of %s rejected before any request went out: %s", url, verdict) return litellm.ImageFetchError( @@ -81,7 +91,7 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr ) -async def async_convert_url_to_base64(url: str) -> str: +async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str: if url.startswith("data:") and ";base64," in url: return url @@ -96,9 +106,9 @@ async def async_convert_url_to_base64(url: str) -> str: return cached_result client: Final = litellm.module_level_aclient - for _ in range(3): + for attempt_timeout in _attempt_timeouts(timeout): try: - response = await async_safe_get(client, url) + response = await async_safe_get(client, url, timeout=attempt_timeout) return _process_image_response(response, url) except litellm.ImageFetchError: raise @@ -109,7 +119,7 @@ async def async_convert_url_to_base64(url: str) -> str: raise litellm.ImageFetchError(f"Error: Unable to fetch image from URL after 3 attempts. url={url}") -def convert_url_to_base64(url: str) -> str: +def convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str: if url.startswith("data:") and ";base64," in url: return url @@ -124,9 +134,9 @@ def convert_url_to_base64(url: str) -> str: return cached_result client: Final = litellm.module_level_client - for _ in range(3): + for attempt_timeout in _attempt_timeouts(timeout): try: - response = safe_get(client, url) + response = safe_get(client, url, timeout=attempt_timeout) return _process_image_response(response, url) except litellm.ImageFetchError: raise @@ -286,14 +296,16 @@ def _inline_message( return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined -async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore) -> str: +async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str: async with in_flight: - return await async_convert_url_to_base64(url) + return await async_convert_url_to_base64(url, timeout=timeout) -async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]: +async def async_convert_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> tuple[str, ...]: in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight)) for url in remote_urls) + fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls) try: return tuple(await asyncio.gather(*fetches)) except BaseException: @@ -329,6 +341,6 @@ async def async_inline_remote_media( remote_urls: Final = _remote_urls_to_inline(messages, should_inline) if not remote_urls: return messages - data_urls: Final = await _fetch_data_urls(remote_urls) + data_urls: Final = await async_convert_urls_to_base64(remote_urls) inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))) return [_inline_message(message, inlined, should_inline) for message in messages] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 5bdb6ee3c18..f7eec2b65ce 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,6 +1,8 @@ import json from abc import ABC -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence +from dataclasses import replace +from itertools import chain from types import MappingProxyType from typing import Final @@ -16,6 +18,8 @@ from litellm.types.decisions import ( ChoiceQuestion, DecisionAnswer, DecisionQuestion, + DecisionsBase64Image, + DecisionsImage, DecisionsIRAnswer, DecisionsIRChoiceAnswer, DecisionsIRChoiceOption, @@ -53,9 +57,8 @@ from litellm.types.utils import is_decisions_model_mode _PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) -_TEXT_ONLY: Final = UnsupportedDecisionsRequest( - reason="input_image content parts are not supported because System One providers accept text input only" -) +_REMOTE_URL_PREFIXES: Final = ("http://", "https://") +_DATA_URL_PREFIX: Final = "data:" def decisions_text(value: DecisionsJSON) -> str: @@ -101,39 +104,153 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest: return DecisionsIRRequest( - input=DecisionsIRState(state=request.state), + input=DecisionsIRState(state=request.state, images=tuple(request.images or ())), audio=request.audio, videos=request.videos, questions=tuple(_ir_question(name, question) for name, question in request.questions.items()), ) -def _has_image(message: OpenAIDecisionInputMessage) -> bool: - return not isinstance(message.content, str) and any( - isinstance(part, OpenAIDecisionInputImage) for part in message.content +def _image_urls(message: OpenAIDecisionInputMessage) -> tuple[str, ...]: + if isinstance(message.content, str): + return () + return tuple(part.image_url for part in message.content if isinstance(part, OpenAIDecisionInputImage)) + + +def input_images(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[DecisionsImage, ...]: + match decision_input: + case DecisionsIRState(): + return decision_input.images + case DecisionsIRMessages(): + return tuple(chain.from_iterable(_image_urls(message) for message in decision_input.messages)) + case _: + assert_never(decision_input) + + +def image_param(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | None: + if not input_images(decision_input): + return None + match decision_input: + case DecisionsIRState(): + return "images" + case DecisionsIRMessages(): + return "input_image" + case _: + assert_never(decision_input) + + +def _kept_part(part: OpenAIDecisionInputText | OpenAIDecisionInputImage, keep: Callable[[str], bool]) -> bool: + return isinstance(part, OpenAIDecisionInputText) or keep(part.image_url) + + +def _message_with_images_where( + message: OpenAIDecisionInputMessage, keep: Callable[[str], bool] +) -> OpenAIDecisionInputMessage | None: + if isinstance(message.content, str): + return message + kept_parts: Final = tuple(part for part in message.content if _kept_part(part, keep)) + return message.model_copy(update={"content": kept_parts}) if kept_parts else None + + +def with_images_where( + decision_input: DecisionsIRState | DecisionsIRMessages, keep: Callable[[DecisionsImage], bool] +) -> DecisionsIRState | DecisionsIRMessages: + match decision_input: + case DecisionsIRState(): + return replace(decision_input, images=tuple(image for image in decision_input.images if keep(image))) + case DecisionsIRMessages(): + return DecisionsIRMessages( + messages=tuple( + kept + for message in decision_input.messages + if (kept := _message_with_images_where(message, keep)) is not None + ) + ) + case _: + assert_never(decision_input) + + +def remote_image_urls(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + image + for image in input_images(decision_input) + if isinstance(image, str) and image.lower().startswith(_REMOTE_URL_PREFIXES) + ) ) -def _message_text(message: OpenAIDecisionInputMessage) -> str: +def _inlined_image(image: DecisionsImage, data_urls: Mapping[str, str]) -> DecisionsImage: + return data_urls.get(image, image) if isinstance(image, str) else image + + +def _inlined_part( + part: OpenAIDecisionInputText | OpenAIDecisionInputImage, data_urls: Mapping[str, str] +) -> OpenAIDecisionInputText | OpenAIDecisionInputImage: + if isinstance(part, OpenAIDecisionInputImage) and part.image_url in data_urls: + return part.model_copy(update={"image_url": data_urls[part.image_url]}) + return part + + +def _inlined_message(message: OpenAIDecisionInputMessage, data_urls: Mapping[str, str]) -> OpenAIDecisionInputMessage: if isinstance(message.content, str): - return message.content - return "\n\n".join(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText)) + return message + return message.model_copy(update={"content": tuple(_inlined_part(part, data_urls) for part in message.content)}) -def _systemone_state( - decision_input: DecisionsIRState | DecisionsIRMessages, -) -> DecisionsJSON | UnsupportedDecisionsRequest: +def with_inlined_images( + decision_input: DecisionsIRState | DecisionsIRMessages, data_urls: Mapping[str, str] +) -> DecisionsIRState | DecisionsIRMessages: + match decision_input: + case DecisionsIRState(): + return replace( + decision_input, images=tuple(_inlined_image(image, data_urls) for image in decision_input.images) + ) + case DecisionsIRMessages(): + return DecisionsIRMessages( + messages=tuple(_inlined_message(message, data_urls) for message in decision_input.messages) + ) + case _: + assert_never(decision_input) + + +def _message_texts(message: OpenAIDecisionInputMessage) -> tuple[str, ...]: + if isinstance(message.content, str): + return (message.content,) + return tuple(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText)) + + +def _systemone_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> DecisionsJSON: match decision_input: case DecisionsIRState(): return decision_input.state case DecisionsIRMessages(): - if any(_has_image(message) for message in decision_input.messages): - return _TEXT_ONLY - return "\n\n".join(_message_text(message) for message in decision_input.messages) + return "\n\n".join(chain.from_iterable(_message_texts(message) for message in decision_input.messages)) case _: assert_never(decision_input) +def _media_type(content_type: str) -> str: + return content_type.split(";", 1)[0].strip().lower() + + +def image_media_type(image: DecisionsImage) -> str | None: + if isinstance(image, DecisionsBase64Image): + return _media_type(image.content_type) + if not image.lower().startswith(_DATA_URL_PREFIX): + return None + return _media_type(image[len(_DATA_URL_PREFIX) :]) + + +def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]: + if isinstance(image, DecisionsBase64Image): + return {"content_type": image_media_type(image), "base64": image.base64} + media_type: Final = image_media_type(image) + if media_type is None: + return image + return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}" + + def _optional(key: str, value: object) -> Mapping[str, object]: return {} if value is None else {key: value} @@ -177,9 +294,8 @@ def ir_to_systemone_request( if request.videos and not supports_video: return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") state: Final = _systemone_state(request.input) - if isinstance(state, UnsupportedDecisionsRequest): - return state keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) + images: Final = input_images(request.input) return { "model": model, "state": state, @@ -194,6 +310,7 @@ def ir_to_systemone_request( else {} ), "questions": {key: _systemone_question(question) for key, question in keyed_questions}, + **({"images": [_systemone_image(image) for image in images]} if images else {}), } @@ -328,6 +445,12 @@ class BaseDecisionsConfig(ABC): api_base_env: tuple[str, ...] = () api_key_required: bool = True supports_safety_identifier: bool = False + supports_images: bool = False + max_images: int | None = None + max_image_bytes: int | None = None + max_total_image_bytes: int | None = None + image_media_types: frozenset[str] | None = None + inlines_remote_images: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index fa8c68ad3a4..967935edfb4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -15,6 +15,12 @@ _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapte class CloudflareDecisionsConfig(BaseDecisionsConfig): api_key_env = ("CLOUDFLARE_API_KEY",) api_base_env = ("CLOUDFLARE_API_BASE",) + supports_images = True + max_images = 4 + max_image_bytes = 4 * 1024 * 1024 + max_total_image_bytes = 8 * 1024 * 1024 + image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) + inlines_remote_images = True def get_default_api_base(self) -> str | None: account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 134ab96ab56..d81f36258c9 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1489,6 +1489,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, body: Mapping[str, object], + logged_body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], @@ -1498,7 +1499,7 @@ class BaseLLMHTTPHandler: data: Final = dict(body) if logging_obj is not None: logging_obj.pre_call( - input=data, + input=dict(logged_body), api_key=api_key, model=model, additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers}, @@ -1513,6 +1514,7 @@ class BaseLLMHTTPHandler: provider_config: BaseDecisionsConfig, request: DecisionsIRRequest, body: Mapping[str, object], + logged_body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], @@ -1525,6 +1527,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, provider_config=provider_config, body=body, + logged_body=logged_body, api_base=api_base, api_key=api_key, headers=headers, @@ -1561,6 +1564,7 @@ class BaseLLMHTTPHandler: provider_config: BaseDecisionsConfig, request: DecisionsIRRequest, body: Mapping[str, object], + logged_body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], @@ -1573,6 +1577,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, provider_config=provider_config, body=body, + logged_body=logged_body, api_base=api_base, api_key=api_key, headers=headers, diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index eb26047d323..a0ffd7f0924 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -8,6 +8,7 @@ import litellm from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( + DecisionsImage, DecisionsIRAnswer, DecisionsIRChoiceAnswer, DecisionsIRChoiceOption, @@ -32,6 +33,9 @@ from litellm.types.decisions import ( OpenAIChoiceQuestion, OpenAIDecisionAnswer, OpenAIDecisionInput, + OpenAIDecisionInputImage, + OpenAIDecisionInputMessage, + OpenAIDecisionInputText, OpenAIDecisionInputTokensDetails, OpenAIDecisionOutputTokensDetails, OpenAIDecisionQuestion, @@ -145,8 +149,23 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: assert_never(question) +def _image_url(image: DecisionsImage) -> str: + return image if isinstance(image, str) else f"data:{image.content_type};base64,{image.base64}" + + +def _state_message(state: DecisionsIRState) -> OpenAIDecisionInputMessage: + return OpenAIDecisionInputMessage( + content=( + *(OpenAIDecisionInputImage(type="input_image", image_url=_image_url(image)) for image in state.images), + OpenAIDecisionInputText(type="input_text", text=decisions_text(state.state)), + ) + ) + + def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | Sequence[Mapping[str, object]]: match decision_input: + case DecisionsIRState() if decision_input.images: + return [_state_message(decision_input).model_dump(mode="json", exclude_none=True)] case DecisionsIRState(): return decisions_text(decision_input.state) case DecisionsIRMessages(): @@ -297,6 +316,7 @@ def ir_to_openai_response( class OpenAIDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" supports_safety_identifier = True + supports_images = True def get_default_api_base(self) -> str | None: return "https://api.openai.com" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3002fafc45..c973375d543 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16029,7 +16029,8 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -16039,7 +16040,8 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "supports_vision": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, @@ -16047,7 +16049,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/clef-flash": { "input_cost_per_token": 9e-08, @@ -16055,7 +16058,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef": { "input_cost_per_token": 2.4e-07, @@ -16063,7 +16067,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef-flash": { "input_cost_per_token": 9e-08, @@ -16071,7 +16076,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 91fae2d1058..479c766b211 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10600,6 +10600,49 @@ "title": "Audio", "type": "array" }, + "images": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded image bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An image/* media type, such as image/png", + "pattern": "^image/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Image", + "type": "object" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Images as data URLs or content_type plus base64, for providers that look at images", + "title": "Images" + }, "model": { "description": "A decision model from the proxy model_list", "title": "Model", @@ -11448,6 +11491,49 @@ "title": "Audio", "type": "array" }, + "images": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded image bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An image/* media type, such as image/png", + "pattern": "^image/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Image", + "type": "object" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Images as data URLs or content_type plus base64, for providers that look at images", + "title": "Images" + }, "model": { "description": "A decision model from the proxy model_list", "title": "Model", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 6b6b7655719..5eb070cb789 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -92,7 +92,7 @@ async def _invalid_request( ) -_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "audio", "videos"}) +_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "audio", "videos"}) def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 4b17f06b562..88a8adc0a1d 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -6,6 +6,7 @@ import httpx from litellm.decisions.call import ( DecisionsAudioClips, DecisionsCall, + DecisionsImages, DecisionsVideoClips, SystemOneQuestions, SystemOneRequestFields, @@ -23,6 +24,7 @@ def _prepare_systemone_call( model: str, state: DecisionsJSON | None, questions: SystemOneQuestions | None, + images: DecisionsImages | None, safety_identifier: str | None, audio: DecisionsAudioClips | None, videos: DecisionsVideoClips | None, @@ -36,7 +38,7 @@ def _prepare_systemone_call( reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields(state=state, questions=questions, audio=audio, videos=videos), + fields=SystemOneRequestFields(state=state, questions=questions, images=images, audio=audio, videos=videos), safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, @@ -65,12 +67,14 @@ async def asystemone( safety_identifier: str | None = None, audio: DecisionsAudioClips | None = None, videos: DecisionsVideoClips | None = None, + images: DecisionsImages | None = None, **kwargs: object, # kwargs-ok: litellm_params forwarded by @client ) -> DecisionsResponse: call: Final = _prepare_systemone_call( model=model, state=state, questions=questions, + images=images, safety_identifier=safety_identifier, audio=audio, videos=videos, @@ -97,12 +101,14 @@ def systemone( safety_identifier: str | None = None, audio: DecisionsAudioClips | None = None, videos: DecisionsVideoClips | None = None, + images: DecisionsImages | None = None, **kwargs: object, # kwargs-ok: litellm_params forwarded by @client ) -> DecisionsResponse: call: Final = _prepare_systemone_call( model=model, state=state, questions=questions, + images=images, safety_identifier=safety_identifier, audio=audio, videos=videos, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index bef61bee40d..6dc4ab78ac8 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -67,6 +67,19 @@ DecisionQuestionMap: TypeAlias = Annotated[ ] +class DecisionsBase64Image(LiteLLMPydanticObjectBase): + content_type: Annotated[str, Field(pattern=r"^image/\S+$", description="An image/* media type, such as image/png")] + base64: Annotated[str, Field(min_length=1, description="Base64-encoded image bytes")] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +DecisionsImage: TypeAlias = ( + Annotated[str, Field(pattern=r"^(?i:data:image/[^;,\s]+(?:;[^;,\s]+)*;base64,.+|https?://\S+)$")] + | DecisionsBase64Image +) + + class DecisionsBase64Audio(LiteLLMPydanticObjectBase): content_type: Annotated[str, Field(pattern=r"^audio/\S+$", description="An audio/* media type, such as audio/wav")] base64: Annotated[str, Field(min_length=1, description="Base64-encoded audio bytes")] @@ -118,6 +131,10 @@ class DecisionsRequestBody(DecisionsMediaBody): DecisionQuestionMap, Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), ] + images: Annotated[ + Sequence[DecisionsImage] | None, + Field(description="Images as data URLs or content_type plus base64, for providers that look at images"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -131,6 +148,7 @@ class DecisionsCallParams(TypedDict, total=False): model: Required[ReadOnly[str]] state: Required[ReadOnly[DecisionsJSON]] questions: Required[ReadOnly[DecisionQuestionMap]] + images: ReadOnly[Sequence[DecisionsImage] | None] audio: ReadOnly[Sequence[DecisionsAudio]] videos: ReadOnly[Sequence[DecisionsVideo]] api_key: ReadOnly[str | None] @@ -419,6 +437,7 @@ _NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) @dataclass(frozen=True, slots=True) class DecisionsIRState: state: DecisionsJSON + images: tuple[DecisionsImage, ...] = () @dataclass(frozen=True, slots=True) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3002fafc45..c973375d543 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16029,7 +16029,8 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -16039,7 +16040,8 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "supports_vision": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, @@ -16047,7 +16049,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/clef-flash": { "input_cost_per_token": 9e-08, @@ -16055,7 +16058,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef": { "input_cost_per_token": 2.4e-07, @@ -16063,7 +16067,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/cloudflare/clef-flash": { "input_cost_per_token": 9e-08, @@ -16071,7 +16076,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", + "supports_vision": true }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, diff --git a/tests/integration/providers/test_cloudflare_clef_images_chaos.py b/tests/integration/providers/test_cloudflare_clef_images_chaos.py new file mode 100644 index 00000000000..8c711544993 --- /dev/null +++ b/tests/integration/providers/test_cloudflare_clef_images_chaos.py @@ -0,0 +1,312 @@ +import asyncio +import base64 +import json +import os +import re +import threading +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import unquote, urlsplit, urlunsplit + +import httpx +import psutil +import pytest +import yaml +from integration._support.bedrock_runtime_peer import PNG_1X1 +from integration._support.client import Gateway, eventually, list_value, object_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +Route = Literal["/v1/systemone", "/systemone", "/v1/decisions", "/decisions"] + +_ROUTES: Final[tuple[Route, ...]] = ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions") +_API_KEY: Final = "synthetic-cloudflare-key" +_INPUT_TOKENS: Final = 412 +_PNG_DATA_URL: Final = f"data:image/png;base64,{base64.b64encode(PNG_1X1).decode()}" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = {"red": {"type": "noul", "instructions": "Is the screenshot red?"}} +_OPENAI_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "red", "instructions": "Is the screenshot red?"} +] + + +@dataclass(frozen=True, slots=True) +class _Call: + route: Route + marker: str + images: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + + +def _state(marker: str) -> str: + return f"Is the checkout screenshot red? marker-{marker}" + + +def _probability(marker: str) -> float: + return round(int(marker[:6], 16) / 0x1000000, 7) + + +def _marker_of(request: Request) -> str: + found: Final = _MARKER.search(request.body.decode() or request.target) + assert found is not None, (request.target, request.body[:200]) + return found.group(1) + + +def _clef_reply(request: Request) -> Reply: + reply: Final[dict[str, JsonValue]] = { + "result": { + "model": "clef", + "answers": {"red": {"type": "noul", "noul": _probability(_marker_of(request))}}, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + }, + "success": True, + } + return Reply(body=json.dumps(reply).encode()) + + +def _is_image_fetch(request: Request) -> bool: + return request.method == "GET" and urlsplit(request.target).path == "/image.png" + + +def _body(call: _Call, model: str) -> dict[str, JsonValue]: + if call.route.endswith("systemone"): + return { + "model": model, + "state": _state(call.marker), + "questions": _SYSTEM_ONE_QUESTIONS, + "images": list(call.images), + "num_retries": 0, + } + parts: Final[list[JsonValue]] = [ + {"type": "input_text", "text": _state(call.marker)}, + *({"type": "input_image", "image_url": image} for image in call.images), + ] + return { + "model": model, + "input": [{"role": "user", "content": parts}], + "questions": _OPENAI_QUESTIONS, + "num_retries": 0, + } + + +def _upstream_by_marker(received: Sequence[Request]) -> dict[str, dict[str, JsonValue]]: + posts: Final = tuple(request for request in received if request.method == "POST") + for request in posts: + assert unquote(request.target) == "/ai/run/@cf/cloudflare/clef", request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers + by_marker: Final = {_marker_of(request): _JSON_OBJECT.validate_json(request.body) for request in posts} + assert len(by_marker) == len(posts), "a request reached clef twice" + return by_marker + + +def _answered_probability(route: Route, answers: JsonValue) -> JsonValue: + if route.endswith("systemone"): + return object_value(object_value(answers)["red"])["noul"] + (answer,) = list_value(answers) + return object_value(answer)["probability"] + + +def _assert_answered(served: _Served) -> None: + assert served.status == 200, served.text + answers: Final = _JSON_OBJECT.validate_json(served.text)["answers"] + assert _answered_probability(served.call.route, answers) == _probability(served.call.marker), served.text + + +def _assert_clef_got_inlined_images(call: _Call, body: dict[str, JsonValue]) -> None: + assert body == { + "model": "clef", + "state": _state(call.marker), + "questions": _SYSTEM_ONE_QUESTIONS, + "images": [_PNG_DATA_URL] * len(call.images), + }, (call, body) + + +async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: + response: Final = await client.post(call.route, json=_body(call, model), headers={"Authorization": f"Bearer {key}"}) + return _Served(call=call, status=response.status_code, text=response.text) + + +async def _burst( + gateway: Gateway, model: str, calls: Sequence[_Call], *, tolerate_transport_errors: bool = False +) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=str(gateway.client.base_url), timeout=90, trust_env=False) as client: + results: Final = await asyncio.gather( + *(_send(client, gateway.key, model, call) for call in calls), return_exceptions=tolerate_transport_errors + ) + for result in results: + assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result) + return tuple(result for result in results if isinstance(result, _Served)) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _config(wire: Wire, tmp_path: Path) -> tuple[Path, str]: + model: Final = f"clef-images-chaos-{uuid.uuid4().hex[:8]}" + config: Final = _JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + path: Final = tmp_path / "clef-images-chaos.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + { + "model_name": model, + "litellm_params": {"model": "cloudflare/clef", "api_base": wire.url, "api_key": _API_KEY}, + } + ], + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + return path, model + + +def _successful_spend_rows(model: str, expected: int) -> int: + rows: Final = eventually( + lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: sum(row["status"] == "success" for row in found) >= expected, + seconds=70, + ) + assert len({row["request_id"] for row in rows}) == len(rows), rows + return sum(row["status"] == "success" for row in rows) + + +def _open_upstream_connections(pid: int, upstream: str) -> int: + port: Final = urlsplit(upstream).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +def _mixed_calls(wire: Wire, count: int) -> tuple[_Call, ...]: + def images(index: int, marker: str) -> tuple[str, ...]: + remote: Final = f"{wire.url}/image.png?id=marker-{marker}" + return ((_PNG_DATA_URL,), (remote,), (remote, _PNG_DATA_URL, remote, _PNG_DATA_URL))[index % 3] + + markers: Final = tuple(uuid.uuid4().hex for _ in range(count)) + return tuple( + _Call(route=_ROUTES[index % len(_ROUTES)], marker=marker, images=images(index, marker)) + for index, marker in enumerate(markers) + ) + + +def _overflow_calls(wire: Wire, count: int) -> tuple[_Call, ...]: + markers: Final = tuple(uuid.uuid4().hex for _ in range(count)) + return tuple( + _Call( + route=_ROUTES[index % len(_ROUTES)], marker=marker, images=(f"{wire.url}/image.png?id=marker-{marker}",) * 5 + ) + for index, marker in enumerate(markers) + ) + + +def _image_host_and_clef(request: Request) -> Reply: + if _is_image_fetch(request): + return Reply(body=PNG_1X1, content_type="image/png") + return _clef_reply(request) + + +@pytest.mark.timeout(600) +async def test_a_mixed_image_burst_across_two_workers_reaches_clef_once_per_request( + gateway: Gateway, tmp_path: Path +) -> None: + with wire_server(_image_host_and_clef) as wire: + path, model = _config(wire, tmp_path) + calls: Final = _mixed_calls(wire, 24) + overflow: Final = _overflow_calls(wire, 8) + overrides: Final = {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])} + with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned: + served: Final = await _burst(owned.gateway, model, (*calls, *overflow)) + assert len(served) == 32 + overflow_markers: Final = frozenset(call.marker for call in overflow) + for item in served: + if item.call.marker in overflow_markers: + assert item.status == 400, item.text + assert "at most 4 images per request, got 5" in item.text, item.text + else: + _assert_answered(item) + received: Final = wire.drain() + fetched: Final = sorted(_marker_of(request) for request in received if _is_image_fetch(request)) + assert overflow_markers.isdisjoint(fetched), fetched + by_marker: Final = _upstream_by_marker(received) + assert sorted(by_marker) == sorted(call.marker for call in calls) + for call in calls: + _assert_clef_got_inlined_images(call, by_marker[call.marker]) + assert _successful_spend_rows(model, 24) == 24 + + +@pytest.mark.timeout(600) +async def test_worker_sigkill_while_images_are_fetched_leaves_the_sibling_serving( + gateway: Gateway, tmp_path: Path +) -> None: + release: Final = threading.Event() + held_fetches: Final[SimpleQueue[str]] = SimpleQueue() + + def respond(request: Request) -> Reply: + if not _is_image_fetch(request): + return _clef_reply(request) + held_fetches.put(_marker_of(request)) + assert release.wait(timeout=60), "The image fetches were never released" + return Reply(body=PNG_1X1, content_type="image/png") + + with wire_server(respond) as wire: + path, model = _config(wire, tmp_path) + calls: Final = tuple( + _Call(route="/v1/systemone", marker=marker, images=(f"{wire.url}/image.png?id=marker-{marker}",)) + for marker in (uuid.uuid4().hex for _ in range(16)) + ) + overrides: Final = {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])} + with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned: + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + burst: Final = asyncio.create_task(_burst(owned.gateway, model, calls, tolerate_transport_errors=True)) + await asyncio.to_thread(eventually, held_fetches.qsize, lambda size: size == 16, 60) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers}) + assert sum(held_by.values()) == 16, held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + with owned.replacing_worker(victim_pid) as kill: + kill.freeze() + await asyncio.to_thread(kill) + release.set() + served: Final = await burst + assert held_by[survivor_pid] >= 8, held_by + assert len(served) == held_by[survivor_pid], (held_by, len(served)) + for item in served: + _assert_answered(item) + follow_up: Final = _Call(route="/v1/decisions", marker=uuid.uuid4().hex, images=(_PNG_DATA_URL,)) + (answered,) = await _burst(owned.gateway, model, (follow_up,)) + _assert_answered(answered) + by_marker: Final = _upstream_by_marker(wire.drain()) + assert sorted(by_marker) == sorted((*(item.call.marker for item in served), follow_up.marker)) + for item in served: + _assert_clef_got_inlined_images(item.call, by_marker[item.call.marker]) + _assert_clef_got_inlined_images(follow_up, by_marker[follow_up.marker]) diff --git a/tests/integration/providers/test_cloudflare_clef_images_wire.py b/tests/integration/providers/test_cloudflare_clef_images_wire.py new file mode 100644 index 00000000000..bf6d01b2455 --- /dev/null +++ b/tests/integration/providers/test_cloudflare_clef_images_wire.py @@ -0,0 +1,877 @@ +import base64 +import json +import math +import os +import re +import threading +import time +import uuid +from collections.abc import Iterator, Sequence +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final, Literal +from urllib.parse import unquote, urlsplit, urlunsplit + +import httpx +import pytest +import yaml +from integration._support.bedrock_runtime_peer import PNG_1X1 +from integration._support.client import Gateway, eventually, gateway_from_environment, list_value, object_value +from integration._support.database import read_rows +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.timing import assert_within_factor_of_control, timed +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +Route = Literal["/v1/systemone", "/systemone", "/v1/decisions", "/decisions"] + +_ROUTES: Final[tuple[Route, ...]] = ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions") +_API_KEY: Final = "synthetic-cloudflare-key" +_CLEF: Final = "cloudflare/clef" +_ALLOWLISTED_MODEL: Final = "clef-images-allowlisted" +_INPUT_TOKENS: Final = 412 +_PNG_BASE64: Final = base64.b64encode(PNG_1X1).decode() +_PNG_DATA_URL: Final = f"data:image/png;base64,{_PNG_BASE64}" +_JPEG_DATA_URL: Final = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQ==" +_PNG_OBJECT: Final[dict[str, JsonValue]] = {"content_type": "image/png", "base64": _PNG_BASE64} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = {"red": {"type": "noul", "instructions": "Is the screenshot red?"}} +_OPENAI_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "red", "instructions": "Is the screenshot red?"} +] +_SPEND_QUERY: Final = ( + 'SELECT spend, status, call_type, model_group, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = %s" +) +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) +_COLD_OPENAPI_SECONDS: Final = 120 +_SLOW_IMAGE_RELEASE: Final = threading.Event() +_IMAGE_INPUT_RECORDER: Final = """ +import json +import os + +import httpx + +from litellm.integrations.custom_logger import CustomLogger + + +class ImageInputRecorder(CustomLogger): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + payload = {"call_id": kwargs.get("litellm_call_id"), "input": kwargs.get("input")} + async with httpx.AsyncClient(trust_env=False, timeout=10) as client: + await client.post(os.environ["IMAGE_INPUT_SINK"], content=json.dumps(payload, default=str)) + + +instance = ImageInputRecorder() +""" + + +def _marker_of(request: Request) -> str: + found: Final = _MARKER.search(request.body.decode()) + assert found is not None, request.body[:200] + return found.group(1) + + +def _state(marker: str) -> str: + return f"Is the checkout screenshot red? marker-{marker}" + + +def _probability(marker: str) -> float: + return round(int(marker[:6], 16) / 0x1000000, 7) + + +def _clef_name(request: Request) -> str: + return unquote(request.target).rsplit("/", 1)[-1] + + +def _clef_reply(request: Request) -> Reply: + reply: Final[dict[str, JsonValue]] = { + "result": { + "model": _clef_name(request), + "answers": {"red": {"type": "noul", "noul": _probability(_marker_of(request))}}, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + } + return Reply(body=json.dumps(reply).encode()) + + +def _speaks_system_one(route: Route) -> bool: + return route.endswith("systemone") + + +def _openai_input(text: str, images: Sequence[JsonValue]) -> list[JsonValue]: + text_parts: Final[tuple[JsonValue, ...]] = ({"type": "input_text", "text": text},) if text else () + image_parts: Final = tuple({"type": "input_image", "image_url": image} for image in images) + return [{"role": "user", "content": [*text_parts, *image_parts]}] + + +def _body( + route: Route, model: str, marker: str, images: Sequence[JsonValue], **extra: JsonValue +) -> dict[str, JsonValue]: + if _speaks_system_one(route): + return { + "model": model, + "state": _state(marker), + "questions": _SYSTEM_ONE_QUESTIONS, + "images": list(images), + **extra, + } + return {"model": model, "input": _openai_input(_state(marker), images), "questions": _OPENAI_QUESTIONS, **extra} + + +def _text_only_body(route: Route, model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]: + if _speaks_system_one(route): + return {"model": model, "state": _state(marker), "questions": _SYSTEM_ONE_QUESTIONS, **extra} + return {"model": model, "input": _state(marker), "questions": _OPENAI_QUESTIONS, **extra} + + +def _expected_response(route: Route, clef_name: str, marker: str) -> dict[str, JsonValue]: + if _speaks_system_one(route): + return { + "model": clef_name, + "answers": {"red": {"type": "noul", "noul": _probability(marker)}}, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + } + return { + "model": clef_name, + "answers": [{"type": "predicate", "name": "red", "probability": _probability(marker)}], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS, + }, + } + + +def _expected_upstream_body(clef_name: str, state: str, images: Sequence[JsonValue] = ()) -> dict[str, JsonValue]: + return { + "model": clef_name, + "state": state, + "questions": _SYSTEM_ONE_QUESTIONS, + **({"images": list(images)} if images else {}), + } + + +def _upstream_body(request: Request, clef_name: str = "clef") -> dict[str, JsonValue]: + assert request.method == "POST", (request.method, request.target) + assert unquote(request.target) == f"/ai/run/@cf/cloudflare/{clef_name}", request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers + return _JSON_OBJECT.validate_json(request.body) + + +def _decide( + gateway: Gateway, route: Route, model: str, marker: str, images: Sequence[JsonValue], **extra: JsonValue +) -> httpx.Response: + return gateway.request("POST", route, _body(route, model, marker, images, **extra)) + + +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _expected_spend(cost_map_key: str) -> float: + cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + priced: Final = cost_map.get(cost_map_key) + assert priced is not None, f"{cost_map_key} has no cost map entry" + return _INPUT_TOKENS * _number(object_value(priced)["input_cost_per_token"]) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _call_type(route: Route) -> str: + return "asystemone" if _speaks_system_one(route) else "adecisions" + + +def _assert_billed(gateway: Gateway, response: httpx.Response, model: str, route: Route, cost_map_key: str) -> None: + expected: Final = _expected_spend(cost_map_key) + assert math.isclose(float(response.headers["x-litellm-response-cost"]), expected, rel_tol=1e-9), response.headers + call_id: Final = response.headers["x-litellm-call-id"] + row: Final = _spend_row(call_id) + assert (row["status"], row["call_type"], row["model_group"], row["prompt_tokens"], row["completion_tokens"]) == ( + "success", + _call_type(route), + model, + _INPUT_TOKENS, + 0, + ), row + assert math.isclose(_number(row["spend"]), expected, rel_tol=1e-9), row + listed: Final = gateway.request("GET", "/spend/logs", params={"request_id": call_id}) + assert listed.status_code == 200, listed.text + (reported,) = _JSON_ROWS.validate_json(listed.content) + assert (reported["request_id"], reported["model_group"], reported["prompt_tokens"]) == ( + call_id, + model, + _INPUT_TOKENS, + ), reported + assert math.isclose(_number(reported["spend"]), expected, rel_tol=1e-9), reported + + +def _images_for(route: Route) -> list[JsonValue]: + return [_PNG_DATA_URL, _PNG_OBJECT] if _speaks_system_one(route) else [_PNG_DATA_URL, _JPEG_DATA_URL] + + +@pytest.mark.parametrize("route", _ROUTES) +def test_images_reach_clef_in_its_images_field_on_every_route_and_are_billed(gateway: Gateway, route: Route) -> None: + marker: Final = uuid.uuid4().hex + images: Final = _images_for(route) + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + response: Final = _decide(gateway, route, model, marker, images) + assert response.status_code == 200, response.text + assert response.json() == _expected_response(route, "clef", marker) + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), images) + _assert_billed(gateway, response, model, route, "cloudflare/@cf/cloudflare/clef") + + +@pytest.mark.parametrize( + ("spelling", "clef_name"), + (("cloudflare/clef-flash", "clef-flash"), ("cloudflare/@cf/cloudflare/clef-omni", "clef-omni")), + ids=("clef-flash", "clef-omni"), +) +def test_clef_flash_and_clef_omni_take_images_and_carry_their_own_price( + gateway: Gateway, spelling: str, clef_name: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=spelling, api_base=wire.url, api_key=_API_KEY) + response: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL]) + assert response.status_code == 200, response.text + assert response.json() == _expected_response("/v1/systemone", clef_name, marker) + (request,) = wire.drain() + assert _upstream_body(request, clef_name) == _expected_upstream_body(clef_name, _state(marker), [_PNG_DATA_URL]) + _assert_billed(gateway, response, model, "/v1/systemone", spelling) + + +def test_an_image_only_message_reaches_clef_with_an_empty_state(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + reply: Final = json.dumps( + { + "result": { + "model": "clef", + "answers": {"red": {"type": "noul", "noul": _probability(marker)}}, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + }, + "success": True, + } + ).encode() + with wire_server(lambda _: Reply(body=reply)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/decisions", + {"model": model, "input": _openai_input("", [_PNG_DATA_URL]), "questions": _OPENAI_QUESTIONS}, + ) + assert response.status_code == 200, response.text + assert response.json() == _expected_response("/v1/decisions", "clef", marker) + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", "", [_PNG_DATA_URL]) + + +def _luna_reply(request: Request) -> Reply: + reply: Final[dict[str, JsonValue]] = { + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "red", "probability": _probability(_marker_of(request))}], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS, + }, + } + return Reply(body=json.dumps(reply).encode()) + + +def test_an_openai_decision_model_gets_system_one_images_as_input_image_parts(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_luna_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="openai/gpt-6-luna", api_base=wire.url, api_key=_API_KEY) + response: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL, _PNG_OBJECT]) + assert response.status_code == 200, response.text + assert object_value(object_value(response.json()["answers"])["red"])["noul"] == _probability(marker) + (request,) = wire.drain() + assert unquote(request.target) == "/v1/decisions", request.target + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "gpt-6-luna", + "input": [ + { + "role": "user", + "type": "message", + "content": [ + {"type": "input_image", "image_url": _PNG_DATA_URL}, + {"type": "input_image", "image_url": _PNG_DATA_URL}, + {"type": "input_text", "text": _state(marker)}, + ], + } + ], + "questions": _OPENAI_QUESTIONS, + } + + +@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions")) +def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream_call( + gateway: Gateway, route: Route +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + remote: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(5)] + for model in (strict, dropping): + refused = _decide(gateway, route, model, marker, remote) + assert refused.status_code == 400, refused.text + assert "at most 4 images per request, got 5" in refused.text, refused.text + assert wire.drain() == () + four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL] + accepted: Final = _decide(gateway, route, strict, marker, four) + assert accepted.status_code == 200, accepted.text + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), four) + + +_MALFORMED: Final[tuple[tuple[str, JsonValue], ...]] = ( + ("bare_string", _PNG_DATA_URL), + ("number", 5), + ("object_not_list", _PNG_OBJECT), + ("url_object", [{"url": _PNG_DATA_URL}]), + ("chat_image_url", [{"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}]), + ("extra_key", [{**_PNG_OBJECT, "detail": "high"}]), + ("missing_base64", [{"content_type": "image/png"}]), + ("non_image_type", [{"content_type": "text/plain", "base64": "AA=="}]), + ("empty_base64", [{"content_type": "image/png", "base64": ""}]), + ("not_a_url", ["a" * 5000]), + ("non_image_data_url", ["data:text/plain;base64,AA=="]), +) + + +@pytest.mark.parametrize(("label", "images"), _MALFORMED, ids=[label for label, _ in _MALFORMED]) +def test_malformed_images_are_refused_unless_the_deployment_drops_params( + gateway: Gateway, label: str, images: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + body: Final = {**_text_only_body("/v1/systemone", strict, marker), "images": images} + refused: Final = gateway.request("POST", "/v1/systemone", body) + assert refused.status_code == 400, (label, refused.text) + assert "Invalid Decisions request" in refused.text and "images" in refused.text, (label, refused.text) + assert wire.drain() == (), label + dropped: Final = gateway.request("POST", "/v1/systemone", {**body, "model": dropping}) + assert dropped.status_code == 200, (label, dropped.text) + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker)), label + + +def test_a_malformed_image_under_drop_params_keeps_the_well_formed_ones(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + images: Final[list[JsonValue]] = [ + {"content_type": "text/plain", "base64": "AA=="}, + _PNG_DATA_URL, + {"url": _PNG_DATA_URL}, + _PNG_OBJECT, + ] + response: Final = _decide(gateway, "/v1/systemone", dropping, marker, images) + assert response.status_code == 200, response.text + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL, _PNG_OBJECT]) + + +_MALFORMED_IMAGE_URLS: Final = ( + ("not_a_url", "not-an-image"), + ("pdf_data_url", "data:application/pdf;base64,AA=="), + ("ftp_url", "ftp://images.example/a.png"), +) + + +@pytest.mark.parametrize( + ("label", "image_url"), _MALFORMED_IMAGE_URLS, ids=[label for label, _ in _MALFORMED_IMAGE_URLS] +) +def test_a_malformed_input_image_is_refused_unless_the_deployment_drops_params( + gateway: Gateway, label: str, image_url: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + refused: Final = _decide(gateway, "/v1/decisions", strict, marker, [image_url, _PNG_DATA_URL]) + assert refused.status_code == 400, (label, refused.text) + assert "Invalid Decisions request: 1 input_image" in refused.text, (label, refused.text) + assert wire.drain() == (), label + dropped: Final = _decide(gateway, "/v1/decisions", dropping, marker, [image_url, _PNG_DATA_URL]) + assert dropped.status_code == 200, (label, dropped.text) + assert dropped.json() == _expected_response("/v1/decisions", "clef", marker), label + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]), label + + +_GIF_BASE64: Final = "R0lGODlhAQABAAAAACw=" +_PNG_DATA_URL_WITH_NAME: Final = f"data:IMAGE/PNG;name=screen.png;base64,{_PNG_BASE64}" +_WEBP_DATA_URL: Final = "data:image/webp;base64,UklGRhoAAABXRUJQ" + + +@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions")) +def test_clef_gets_png_jpeg_and_webp_images_without_media_type_parameters(gateway: Gateway, route: Route) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + images: Final = [_PNG_DATA_URL_WITH_NAME, _JPEG_DATA_URL, _WEBP_DATA_URL] + response: Final = _decide(gateway, route, model, marker, images) + assert response.status_code == 200, response.text + assert response.json() == _expected_response(route, "clef", marker) + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body( + "clef", _state(marker), [_PNG_DATA_URL, _JPEG_DATA_URL, _WEBP_DATA_URL] + ) + + +def test_an_openai_decision_model_gets_a_data_url_with_media_type_parameters_unchanged(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_luna_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="openai/gpt-6-luna", api_base=wire.url, api_key=_API_KEY) + response: Final = _decide(gateway, "/v1/decisions", model, marker, [_PNG_DATA_URL_WITH_NAME]) + assert response.status_code == 200, response.text + (request,) = wire.drain() + sent: Final = list_value( + object_value(list_value(_JSON_OBJECT.validate_json(request.body)["input"])[0])["content"] + ) + assert sent == [ + {"type": "input_text", "text": _state(marker)}, + {"type": "input_image", "image_url": _PNG_DATA_URL_WITH_NAME}, + ] + + +_REFUSED_TYPES: Final[tuple[tuple[str, Route, JsonValue], ...]] = ( + ("gif_data_url", "/v1/decisions", f"data:image/gif;base64,{_GIF_BASE64}"), + ("gif_object", "/v1/systemone", {"content_type": "image/gif", "base64": _GIF_BASE64}), +) + + +@pytest.mark.parametrize(("label", "route", "image"), _REFUSED_TYPES, ids=[label for label, _, _ in _REFUSED_TYPES]) +def test_an_image_type_clef_refuses_is_refused_unless_the_deployment_drops_params( + gateway: Gateway, label: str, route: Route, image: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + refused: Final = _decide(gateway, route, strict, marker, [image, _PNG_DATA_URL]) + assert refused.status_code == 400, (label, refused.text) + assert ( + "cloudflare takes only image/jpeg, image/jpg, image/png, image/webp images, got image/gif" in refused.text + ), (label, refused.text) + assert wire.drain() == (), label + dropped: Final = _decide(gateway, route, dropping, marker, [image, _PNG_DATA_URL]) + assert dropped.status_code == 200, (label, dropped.text) + assert dropped.json() == _expected_response(route, "clef", marker), label + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]), label + + +@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions")) +def test_an_image_clef_refuses_does_not_count_toward_its_limit_when_the_deployment_drops_params( + gateway: Gateway, route: Route +) -> None: + marker: Final = uuid.uuid4().hex + four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL] + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + accepted: Final = _decide(gateway, route, dropping, marker, [f"data:image/gif;base64,{_GIF_BASE64}", *four]) + assert accepted.status_code == 200, accepted.text + assert accepted.json() == _expected_response(route, "clef", marker) + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), four) + + +_MIB: Final = 1024 * 1024 + + +def _png_data_url(decoded_bytes: int) -> str: + return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}" + + +def _png_object(decoded_bytes: int) -> dict[str, JsonValue]: + return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} + + +_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], str], ...]] = ( + ("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1),), "got one of 4,194,305 bytes"), + ("one_over_4mib", "/v1/decisions", (_png_data_url(4 * _MIB + 1),), "got one of 4,194,305 bytes"), + ( + "total_over_8mib", + "/v1/systemone", + (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)), + "got 8,388,609 bytes", + ), +) + + +@pytest.mark.parametrize( + ("label", "route", "images", "refusal"), _OVERSIZED, ids=[f"{label}{route}" for label, route, _, _ in _OVERSIZED] +) +def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call( + gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], refusal: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + for model in (strict, dropping): + refused = _decide(gateway, route, model, marker, images) + assert refused.status_code == 400, (label, refused.text[:500]) + assert refusal in refused.text, (label, refused.text[:500]) + assert wire.drain() == (), label + at_limits: Final[list[JsonValue]] = [_png_data_url(4 * _MIB), _png_object(4 * _MIB)] + accepted: Final = _decide(gateway, "/v1/systemone", strict, marker, at_limits) + assert accepted.status_code == 200, accepted.text[:500] + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), at_limits), label + + +def _raw_system_one(gateway: Gateway, model: str, marker: str, images_fragment: str) -> httpx.Response: + raw: Final = ( + f'{{"model": {json.dumps(model)}, "state": {json.dumps(_state(marker))}, ' + f'"questions": {json.dumps(_SYSTEM_ONE_QUESTIONS)}, {images_fragment}}}' + ) + return gateway.client.post( + "/v1/systemone", + content=raw.encode(), + headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}, + ) + + +def test_a_duplicated_images_key_is_judged_by_its_last_value(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + malformed: Final = json.dumps([{"content_type": "text/plain", "base64": "AA=="}]) + good: Final = json.dumps([_PNG_DATA_URL]) + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + refused: Final = _raw_system_one(gateway, model, marker, f'"images": {good}, "images": {malformed}') + assert refused.status_code == 400, refused.text + assert "Invalid Decisions request" in refused.text, refused.text + assert wire.drain() == () + accepted: Final = _raw_system_one(gateway, model, marker, f'"images": {malformed}, "images": {good}') + assert accepted.status_code == 200, accepted.text + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]) + + +def test_images_next_to_input_are_refused_unless_params_are_dropped(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True) + refused: Final = gateway.request( + "POST", "/v1/decisions", _text_only_body("/v1/decisions", strict, marker, images=[_PNG_DATA_URL]) + ) + assert refused.status_code == 400, refused.text + assert "images belongs to the System One format" in refused.text, refused.text + assert wire.drain() == () + for accepted in ( + gateway.request( + "POST", "/v1/decisions", _text_only_body("/v1/decisions", dropping, marker, images=[_PNG_DATA_URL]) + ), + gateway.request( + "POST", + "/v1/decisions", + _text_only_body("/v1/decisions", strict, marker, images=[_PNG_DATA_URL], drop_params=True), + ), + ): + assert accepted.status_code == 200, accepted.text + assert accepted.json() == _expected_response("/v1/decisions", "clef", marker) + assert [_upstream_body(request) for request in wire.drain()] == [ + _expected_upstream_body("clef", _state(marker)) + ] * 2 + + +def test_a_remote_image_on_the_shared_proxy_is_refused_before_any_fetch(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + for route in ("/v1/systemone", "/v1/decisions"): + refused = _decide(gateway, route, model, marker, [f"{wire.url}/image.png?id=marker-{marker}"]) + assert refused.status_code == 400, (route, refused.text) + assert "Unable to fetch image from URL" in refused.text, (route, refused.text) + assert "user_url_allowed_hosts" in refused.text, (route, refused.text) + assert wire.drain() == () + + +def test_a_clef_rejection_of_an_image_reaches_the_caller_like_a_text_only_one(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + message: Final = "Image could not be decoded" + rejection: Final = json.dumps( + {"result": {}, "success": False, "errors": [{"code": 5006, "message": message}], "messages": []} + ).encode() + garbage: Final = "data:image/png;base64," + "a" * 5000 + with wire_server(lambda _: Reply(status=422, body=rejection)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + text_only: Final = gateway.request( + "POST", "/v1/systemone", _text_only_body("/v1/systemone", model, marker, num_retries=0) + ) + with_image: Final = _decide(gateway, "/v1/systemone", model, marker, [garbage], num_retries=0) + for response in (text_only, with_image): + assert response.status_code >= 400, response.text + assert message in response.text and "answers" not in response.text, response.text + assert with_image.status_code == text_only.status_code, (text_only.status_code, with_image.status_code) + first, second = (_upstream_body(request) for request in wire.drain()) + assert first == _expected_upstream_body("clef", _state(marker)) + assert second == _expected_upstream_body("clef", _state(marker), [garbage]) + + +def test_an_unauthenticated_image_request_is_refused_on_every_route(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + for route in _ROUTES: + response = gateway.client.post(route, json=_body(route, model, marker, [_PNG_DATA_URL])) + assert response.status_code == 401, (route, response.text) + assert "Authentication Error" in response.text, response.text + assert wire.drain() == () + + +def test_unrelated_chat_keeps_working_after_a_run_of_refused_image_requests(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + control: Final = scenario.model() + for _, images in _MALFORMED: + refused = gateway.request( + "POST", "/v1/systemone", {**_text_only_body("/v1/systemone", model, marker), "images": images} + ) + assert refused.status_code == 400, refused.text + assert _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL] * 5).status_code == 400 + assert wire.drain() == () + assert gateway.chat(control)["choices"], "the control deployment stopped answering" + accepted: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL]) + assert accepted.status_code == 200, accepted.text + (request,) = wire.drain() + assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]) + + +def _timed_liveliness(gateway: Gateway) -> tuple[int, float]: + started: Final = time.monotonic() + response: Final = gateway.request("GET", "/health/liveliness") + return response.status_code, time.monotonic() - started + + +_WORST_CASE_IMAGES: Final = ( + ("unterminated_data_url", "DATA:IMAGE/" + "x" * 2_000_000), + ("url_with_trailing_space", "HTTPS://" + "x" * 2_000_000 + " "), + ("parameters_without_base64", "DATA:IMAGE/PNG" + ";x" * 1_000_000), +) + + +@pytest.mark.parametrize(("label", "image"), _WORST_CASE_IMAGES, ids=[label for label, _ in _WORST_CASE_IMAGES]) +def test_the_worst_case_image_string_is_refused_in_time_while_health_keeps_answering( + gateway: Gateway, label: str, image: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clef_reply) as wire, gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=1) as pool: + model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY) + control: Final = timed(lambda: _decide(gateway, "/v1/systemone", model, marker, ["x" * len(image)])) + assert control.value.status_code == 400, control.value.text + liveliness: Final = pool.submit(_timed_liveliness, gateway) + refused: Final = timed(lambda: _decide(gateway, "/v1/systemone", model, marker, [image])) + health_status, health_elapsed = liveliness.result() + assert refused.value.status_code == 400, (label, refused.value.text[:500]) + assert "Invalid Decisions request" in refused.value.text, (label, refused.value.text[:500]) + assert_within_factor_of_control(refused.seconds, control=control.seconds, factor=5) + assert health_status == 200, (health_status, health_elapsed) + assert_within_factor_of_control(health_elapsed, control=control.seconds, factor=5) + assert wire.drain() == () + + +def _image_object_schema(schema: dict[str, JsonValue]) -> dict[str, JsonValue]: + images: Final = object_value(object_value(schema["properties"])["images"]) + array: Final = next( + object_value(branch) for branch in list_value(images["anyOf"]) if object_value(branch).get("type") == "array" + ) + entries: Final = list_value(object_value(array["items"])["anyOf"]) + return next(object_value(entry) for entry in entries if object_value(entry).get("type") == "object") + + +def test_openapi_documents_what_an_image_object_must_carry(gateway: Gateway) -> None: + response: Final = gateway.client.get( + "/openapi.json", headers={"Authorization": f"Bearer {gateway.key}"}, timeout=_COLD_OPENAPI_SECONDS + ) + assert response.status_code == 200, response.text + paths: Final = object_value(_JSON_OBJECT.validate_json(response.content)["paths"]) + for route in ("/v1/systemone", "/systemone"): + post: Final = object_value(object_value(paths[route])["post"]) + content: Final = object_value(object_value(post["requestBody"])["content"]) + schema: Final = object_value(object_value(content["application/json"])["schema"]) + properties: Final = object_value(_image_object_schema(schema)["properties"]) + assert object_value(properties["content_type"])["pattern"] == r"^image/\S+$", (route, properties) + assert object_value(properties["base64"])["minLength"] == 1, (route, properties) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _image_host_and_clef(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if request.method == "GET" and path == "/image.png": + return Reply(body=PNG_1X1, content_type="image/png") + if request.method == "GET" and path == "/page.html": + return Reply(body=b"", content_type="text/html; charset=utf-8") + if request.method == "GET" and path == "/slow.png": + _SLOW_IMAGE_RELEASE.wait(timeout=30) + return Reply(body=PNG_1X1, content_type="image/png") + if request.method == "GET": + return Reply(status=404, body=b"not found", content_type="text/plain") + return _clef_reply(request) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def wire() -> Iterator[Wire]: + with wire_server(_image_host_and_clef) as served: + yield served + _SLOW_IMAGE_RELEASE.set() + + +@pytest.fixture(scope="module") +def sink() -> Iterator[Wire]: + with wire_server(lambda _: Reply()) as served: + yield served + + +@pytest.fixture(scope="module") +def allowlisted(rig: Gateway, wire: Wire, sink: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("clef-images-allowlisted") + (directory / "image_input_recorder.py").write_text(_IMAGE_INPUT_RECORDER) + config: Final = _JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + path: Final = directory / "clef-images-allowlisted.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + { + "model_name": _ALLOWLISTED_MODEL, + "litellm_params": {"model": _CLEF, "api_base": wire.url, "api_key": _API_KEY}, + } + ], + "litellm_settings": { + **object_value(config["litellm_settings"]), + "callbacks": ["image_input_recorder.instance"], + }, + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + overrides: Final = { + "DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"]), + "IMAGE_INPUT_SINK": sink.url, + } + with owned_proxy(rig, directory, overrides, config=path) as owned: + yield owned + + +@pytest.fixture(autouse=True) +def _drained(request: pytest.FixtureRequest) -> None: + for name in ("wire", "sink"): + if name in request.fixturenames: + request.getfixturevalue(name).drain() + + +def _routes(received: Sequence[Request]) -> list[tuple[str, str]]: + return [(request.method, urlsplit(unquote(request.target)).path) for request in received] + + +def _logged_inputs(sink: Wire, call_ids: Sequence[str]) -> dict[str, JsonValue]: + eventually(sink.received.qsize, lambda pending: pending >= len(call_ids), seconds=30) + events: Final = tuple(_JSON_OBJECT.validate_json(request.body) for request in sink.drain()) + return {str(event["call_id"]): event["input"] for event in events} + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions")) +def test_an_allowlisted_remote_image_reaches_clef_as_bytes_while_callbacks_log_the_url_the_caller_sent( + allowlisted: Gateway, wire: Wire, sink: Wire, route: Route +) -> None: + marker: Final = uuid.uuid4().hex + image_url: Final = f"{wire.url}/image.png?id=marker-{marker}" + responses: Final = tuple(_decide(allowlisted, route, _ALLOWLISTED_MODEL, marker, [image_url]) for _ in range(2)) + for response in responses: + assert response.status_code == 200, response.text + assert response.json() == _expected_response(route, "clef", marker) + received: Final = wire.drain() + posts: Final = tuple(request for request in received if request.method == "POST") + assert {target for method, target in _routes(received) if method == "GET"} == {"/image.png"}, _routes(received) + assert [_upstream_body(request) for request in posts] == [ + _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]) + ] * 2 + call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses) + assert len(set(call_ids)) == 2, call_ids + assert _logged_inputs(sink, call_ids) == { + call_id: _expected_upstream_body("clef", _state(marker), [image_url]) for call_id in call_ids + } + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_missing_remote_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None: + marker: Final = uuid.uuid4().hex + response: Final = _decide( + allowlisted, "/v1/systemone", _ALLOWLISTED_MODEL, marker, [f"{wire.url}/missing.png?id=marker-{marker}"] + ) + assert response.status_code == 400, response.text + assert "Unable to fetch image from URL. Status code: 404" in response.text, response.text + assert _routes(wire.drain()) == [("GET", "/missing.png")] + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_remote_page_that_is_not_an_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None: + marker: Final = uuid.uuid4().hex + response: Final = _decide( + allowlisted, "/v1/decisions", _ALLOWLISTED_MODEL, marker, [f"{wire.url}/page.html?id=marker-{marker}"] + ) + assert response.status_code == 400, response.text + assert "images, got text/html" in response.text, response.text + assert _routes(wire.drain()) == [("GET", "/page.html")] + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_hanging_image_host_gets_one_attempt_within_the_callers_timeout(allowlisted: Gateway, wire: Wire) -> None: + marker: Final = uuid.uuid4().hex + response: Final = _decide( + allowlisted, + "/v1/systemone", + _ALLOWLISTED_MODEL, + marker, + [f"{wire.url}/slow.png?id=marker-{marker}"], + timeout=2, + num_retries=0, + ) + received: Final = wire.drain() + _SLOW_IMAGE_RELEASE.set() + assert response.status_code == 400, response.text + assert "Unable to fetch image from URL after 3 attempts" in response.text, response.text + assert _routes(received) == [("GET", "/slow.png")] diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 64387b293ce..d989a0c49aa 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -116,6 +116,7 @@ class _Provider: wraps_result: bool cost_map_key: str | None speaks_openai: bool = False + takes_images: bool = False ai_decide: bool = False provider_reported_cost: float | None = None @@ -184,6 +185,7 @@ _PROVIDERS: Final = ( _API_KEY, True, "cloudflare/@cf/cloudflare/clef", + takes_images=True, ), _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), _Provider( @@ -211,9 +213,11 @@ _PROVIDERS: Final = ( _Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True), ) _SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) +_TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images) _PERPLEXITY: Final = _PROVIDERS[0] _TYPESAFE: Final = _PROVIDERS[1] _OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) +_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images) _PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"input": _INPUT}), @@ -235,12 +239,13 @@ _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}), ("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}), ) +_PNG_DATA_URL: Final = "data:image/png;base64,iVBORw0KGgo=" _IMAGE_INPUT: Final[list[JsonValue]] = [ { "role": "user", "content": [ {"type": "input_text", "text": _INPUT}, - {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "auto"}, ], } ] @@ -250,7 +255,7 @@ _OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [ "type": "message", "content": [ {"type": "input_text", "text": _INPUT}, - {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "auto"}, ], } ] @@ -337,6 +342,13 @@ def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: d assert "Invalid Decisions request" in response.text, (label, response.text) +def _decide_with_images_in_both_formats(gateway: Gateway, model: str) -> tuple[httpx.Response, httpx.Response]: + return ( + _decide(gateway, model, input=_IMAGE_INPUT), + _decide_in_system_one_format(gateway, model, images=[_PNG_DATA_URL]), + ) + + def _assert_refused_for_safety_identifier(response: httpx.Response) -> None: assert response.status_code == 400, response.text assert "safety_identifier" in response.text and "drop_params" in response.text, response.text @@ -462,18 +474,37 @@ def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gate assert _upstream_calls(gateway, handle) == [] -@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) -def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call( +@pytest.mark.parametrize("provider", _TEXT_ONLY_PROVIDERS, ids=_provider_id) +def test_text_only_providers_refuse_images_unless_the_deployment_drops_params( gateway: Gateway, provider: _Provider ) -> None: with gateway.scenario() as scenario: handle: Final = _register(scenario, provider.upstream_reply()) - model: Final = _deployment(scenario, handle, provider) - response: Final = _decide(gateway, model, input=_IMAGE_INPUT) - assert response.status_code == 400, response.text - assert "input_image" in response.text + strict: Final = _deployment(scenario, handle, provider) + openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, strict) + assert (openai_format.status_code, system_one_format.status_code) == (400, 400), openai_format.text + assert "['input_image']" in openai_format.text and "drop_params" in openai_format.text, openai_format.text + assert "['images']" in system_one_format.text, system_one_format.text assert _upstream_calls(gateway, handle) == [] + dropping: Final = _deployment(scenario, handle, provider, drop_params=True) + accepted: Final = _decide_with_images_in_both_formats(gateway, dropping) + assert [response.status_code for response in accepted] == [200, 200], [response.text for response in accepted] + assert [call["body"] for call in _upstream_calls(gateway, handle)] == [provider.upstream_body()] * 2 + + +def test_cloudflare_receives_images_from_either_format_in_its_images_field(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _CLOUDFLARE.upstream_reply()) + model: Final = _deployment(scenario, handle, _CLOUDFLARE) + openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model) + assert openai_format.status_code == 200, openai_format.text + assert openai_format.json() == _CLOUDFLARE.litellm_response() + assert system_one_format.status_code == 200, system_one_format.text + assert system_one_format.json()["answers"] == _SYSTEM_ONE_ANSWERS + expected_body: Final = {**_CLOUDFLARE.upstream_body(), "images": [_PNG_DATA_URL]} + assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2 + def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: with gateway.scenario() as scenario: diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e36d5b8f9b1..6750328abac 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1,10 +1,11 @@ from __future__ import annotations import asyncio +import base64 import json import re import threading -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from concurrent.futures import wait from types import MappingProxyType from typing import Final @@ -22,6 +23,7 @@ from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.decisions import ( ChoiceAnswer, DecisionsResponse, @@ -1539,20 +1541,16 @@ async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_ans ("model", "request_kwargs", "message"), ( ( - "typesafe/jev-1.13", + "openai/gpt-6-luna", { - "input": [ - { - "role": "user", - "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], - } - ], + "input": "review", + "images": ["data:image/png;base64,AA=="], "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], }, - "cannot serve this request", + "send images as input_image content parts", ), ), - ids=("image_to_systemone_provider",), + ids=("images_and_input",), ) async def test_requests_a_provider_cannot_serve_are_rejected_before_http( respx_mock: respx.MockRouter, @@ -1735,6 +1733,776 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_ } +_CLOUDFLARE_RUN_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare" +_PERPLEXITY_DECISIONS_URL: Final = "https://api.perplexity.ai/v1/decisions" +_PNG_DATA_URL: Final = "data:image/png;base64,AA==" +_REMOTE_IMAGE_URL: Final = "https://images.example/screen.png" +_SCREEN_TEXT: Final = "Is the screen cracked?" +_SYSTEMONE_PREDICATE: Final[Mapping[str, object]] = MappingProxyType( + {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}} +) + + +def _screen_messages(image_url: str) -> tuple[Mapping[str, object], ...]: + return ( + { + "role": "user", + "content": [ + {"type": "input_text", "text": _SCREEN_TEXT}, + {"type": "input_image", "image_url": image_url, "detail": "high"}, + ], + }, + ) + + +def _image_requests(image_url: str) -> tuple[tuple[str, Mapping[str, object]], ...]: + return ( + ("decisions", MappingProxyType({"input": _screen_messages(image_url), "questions": _PREDICATE_QUESTIONS})), + ( + "systemone", + MappingProxyType({"state": _SCREEN_TEXT, "questions": _SYSTEMONE_PREDICATE, "images": [image_url]}), + ), + ) + + +@pytest.fixture +def cloudflare_account(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("model", ("clef", "clef-flash")) +async def test_input_images_reach_cloudflare_clef_as_its_images_field(model: str, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/{model}").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model=f"cloudflare/{model}", + input=( + *_screen_messages(_PNG_DATA_URL), + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/webp;base64,BB=="}]}, + {"role": "user", "content": "Order 1234."}, + ), + questions=_PREDICATE_QUESTIONS, + ) + + assert json.loads(route.calls[0].request.content) == { + "model": model, + "state": f"{_SCREEN_TEXT}\n\nOrder 1234.", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "images": [_PNG_DATA_URL, "data:image/webp;base64,BB=="], + } + assert isinstance(response, OpenAIDecisionResponse) + + +@pytest.mark.usefixtures("cloudflare_account") +def test_systemone_images_reach_cloudflare_clef_in_the_callers_form(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + response: Final = litellm.systemone( + model="cloudflare/clef", + state={"ticket": 1234}, + questions=_SYSTEMONE_PREDICATE, + images=[_PNG_DATA_URL, {"content_type": "image/webp", "base64": "BB=="}], + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "clef", + "state": {"ticket": 1234}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "images": [_PNG_DATA_URL, {"content_type": "image/webp", "base64": "BB=="}], + } + assert isinstance(response, DecisionsResponse) + + +@pytest.mark.asyncio +class _InputRecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.inputs: list[object] = [] + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.inputs.append(kwargs.get("input")) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS) +async def test_logging_callbacks_get_the_image_url_the_caller_sent_while_cloudflare_gets_its_bytes( + api: str, + request_kwargs: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + async_only_image_fetch, +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await getattr(litellm, f"a{api}")(model="cloudflare/clef", **request_kwargs) + await _drain_logging_worker() + + wire_body: Final = json.loads(route.calls[0].request.content) + assert wire_body["images"] == [async_only_image_fetch.data_url] + assert recording_logger.inputs == [{**wire_body, "images": [_REMOTE_IMAGE_URL]}] + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS) +async def test_remote_images_are_fetched_off_the_event_loop_and_inlined_for_cloudflare( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, async_only_image_fetch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await getattr(litellm, f"a{api}")(model="cloudflare/clef", **request_kwargs) + + assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL] + assert json.loads(route.calls[0].request.content)["images"] == [async_only_image_fetch.data_url] + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS) +def test_sync_decisions_fetch_and_inline_remote_images_for_cloudflare( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + png: Final = b"\x89PNG\r\n\x1a\n" + image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(content=png, headers={"content-type": "image/png"}) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + getattr(litellm, api)(model="cloudflare/clef", **request_kwargs) + + assert image_route.call_count == 1 + assert json.loads(route.calls[0].request.content)["images"] == [ + f"data:image/png;base64,{base64.b64encode(png).decode()}" + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync")) +async def test_remote_image_downloads_use_the_callers_timeout( + sync: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport())) + image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond( + content=b"\x89PNG\r\n\x1a\n", headers={"content-type": "image/png"} + ) + respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + request_kwargs: Final = {**_image_requests(_REMOTE_IMAGE_URL)[1][1], "model": "cloudflare/clef", "timeout": 7.0} + + if sync: + litellm.systemone(**request_kwargs) + else: + await litellm.asystemone(**request_kwargs) + + sent_timeout: Final = image_route.calls.last.request.extensions["timeout"] + assert sent_timeout.keys() == httpx.Timeout(7.0).as_dict().keys() + assert all(6.0 < seconds <= 7.0 for seconds in sent_timeout.values()), sent_timeout + + +@pytest.mark.asyncio +async def test_remote_input_images_reach_openai_unfetched( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, async_only_image_fetch +) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await litellm.adecisions( + model="openai/gpt-6-luna", + input=_screen_messages(_REMOTE_IMAGE_URL), + questions=_PREDICATE_QUESTIONS, + api_key="caller-key", + ) + + assert json.loads(route.calls[0].request.content)["input"][0]["content"][1]["image_url"] == _REMOTE_IMAGE_URL + assert async_only_image_fetch.fetched == [] + + +_IMAGE_REFUSALS: Final[tuple[tuple[str, Mapping[str, object], str], ...]] = ( + ("decisions", _image_requests(_PNG_DATA_URL)[0][1], "input_image"), + ("systemone", _image_requests(_PNG_DATA_URL)[1][1], "images"), +) + + +@pytest.mark.parametrize(("api", "request_kwargs", "param"), _IMAGE_REFUSALS, ids=_PREDICATE_REQUEST_IDS) +def test_images_are_refused_before_http_when_the_provider_cannot_take_them( + api: str, + request_kwargs: Mapping[str, object], + param: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: + getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_PNG_DATA_URL), ids=_PREDICATE_REQUEST_IDS) +async def test_images_are_dropped_from_the_wire_under_drop_params( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + await getattr(litellm, f"a{api}")( + model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": _SCREEN_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +_MALFORMED_IMAGE_ENTRIES: Final[tuple[Mapping[str, object] | str, ...]] = ( + {"url": _PNG_DATA_URL}, + {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}, + {"content_type": "image/png", "base64": "AA==", "detail": "high"}, + {"content_type": "text/plain", "base64": "AA=="}, + {"content_type": "image/png", "base64": ""}, + "a" * 5000, + "data:text/plain;base64,AA==", +) +_MALFORMED_IMAGE_ENTRY_IDS: Final = ( + "url_object", + "chat_image_url", + "extra_key", + "non_image_type", + "empty_base64", + "not_a_url", + "non_image_data_url", +) +_MALFORMED_IMAGES: Final[tuple[object, ...]] = (_PNG_DATA_URL, *([entry] for entry in _MALFORMED_IMAGE_ENTRIES)) +_MALFORMED_IMAGE_IDS: Final = ("bare_string", *_MALFORMED_IMAGE_ENTRY_IDS) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("images", _MALFORMED_IMAGES, ids=_MALFORMED_IMAGE_IDS) +def test_malformed_images_are_rejected_before_http( + images: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*images"): + litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images) + + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("images", _MALFORMED_IMAGES, ids=_MALFORMED_IMAGE_IDS) +async def test_malformed_images_are_dropped_from_the_wire_under_drop_params( + images: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.asystemone( + model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images, drop_params=True + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "clef", + "state": _SCREEN_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("entry", _MALFORMED_IMAGE_ENTRIES, ids=_MALFORMED_IMAGE_ENTRY_IDS) +async def test_a_malformed_image_under_drop_params_keeps_the_well_formed_ones( + entry: Mapping[str, object] | str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[entry, _PNG_DATA_URL, {"content_type": "image/png", "base64": "AA=="}], + drop_params=True, + ) + + assert json.loads(route.calls[0].request.content)["images"] == [ + _PNG_DATA_URL, + {"content_type": "image/png", "base64": "AA=="}, + ] + + +@pytest.mark.asyncio +async def test_empty_images_next_to_input_are_ignored( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await litellm.adecisions( + model="openai/gpt-6-luna", + api_key="caller-key", + input="review", + questions=_PREDICATE_QUESTIONS, + images=[], + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + + +@pytest.mark.asyncio +async def test_images_next_to_input_are_dropped_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await litellm.adecisions( + model="openai/gpt-6-luna", + api_key="caller-key", + input="review", + questions=_PREDICATE_QUESTIONS, + images=[_PNG_DATA_URL], + drop_params=True, + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + + +def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, object]], ...]: + return ( + ( + "decisions", + MappingProxyType( + { + "input": ( + { + "role": "user", + "content": [{"type": "input_image", "image_url": _PNG_DATA_URL} for _ in range(count)], + }, + ), + "questions": _PREDICATE_QUESTIONS, + } + ), + ), + ( + "systemone", + MappingProxyType( + {"state": _SCREEN_TEXT, "questions": _SYSTEMONE_PREDICATE, "images": [_PNG_DATA_URL] * count} + ), + ), + ) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _cloudflare_image_requests(5), ids=_PREDICATE_REQUEST_IDS) +def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: + getattr(litellm, api)(model="cloudflare/clef", **request_kwargs) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _cloudflare_image_requests(4), ids=_PREDICATE_REQUEST_IDS) +def test_the_four_images_cloudflare_takes_reach_it( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + getattr(litellm, api)(model="cloudflare/clef", **request_kwargs) + + assert json.loads(route.calls[0].request.content)["images"] == [_PNG_DATA_URL] * 4 + + +@pytest.mark.asyncio +async def test_dropping_the_only_input_images_is_rejected_before_http( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"\['input_image'\].*leaves no input") as caught: + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + api_key="caller-key", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}], + questions=_PREDICATE_QUESTIONS, + drop_params=True, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +_MALFORMED_IMAGE_URLS: Final = ("not-an-image", "data:application/pdf;base64,AA==", "ftp://images.example/a.png") +_MALFORMED_IMAGE_URL_IDS: Final = ("not_a_url", "pdf_data_url", "ftp_url") +_OPENAI_DECISIONS_URL: Final = "https://api.openai.com/v1/decisions" + + +@pytest.fixture +def openai_default_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + + +@pytest.mark.usefixtures("cloudflare_account", "openai_default_base") +@pytest.mark.parametrize( + ("model", "upstream"), + (("cloudflare/clef", f"{_CLOUDFLARE_RUN_URL}/clef"), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)), + ids=("clef", "luna"), +) +@pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS) +def test_malformed_input_image_urls_are_rejected_before_http( + model: str, upstream: str, image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(upstream).respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 1 input_image") as caught: + litellm.decisions( + model=model, api_key="caller-key", input=_screen_messages(image_url), questions=_PREDICATE_QUESTIONS + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS) +async def test_a_malformed_input_image_under_drop_params_keeps_the_text_and_well_formed_images( + image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + input=( + *_screen_messages(image_url), + {"role": "user", "content": [{"type": "input_image", "image_url": image_url}]}, + {"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}, + ), + questions=_PREDICATE_QUESTIONS, + drop_params=True, + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "clef", + "state": _SCREEN_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "images": [_PNG_DATA_URL], + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_dropping_the_only_malformed_input_images_is_rejected_before_http( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 2 input_image") as caught: + await litellm.adecisions( + model="cloudflare/clef", + input=[ + {"role": "user", "content": [{"type": "input_image", "image_url": url}]} + for url in _MALFORMED_IMAGE_URLS[:2] + ], + questions=_PREDICATE_QUESTIONS, + drop_params=True, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +_MIB: Final = 1024 * 1024 + + +def _png_data_url(decoded_bytes: int) -> str: + return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}" + + +def _png_object(decoded_bytes: int) -> Mapping[str, object]: + return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()} + + +_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = ( + ((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), + ((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"), + ( + (_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)), + "at most 8,388,608 bytes (8 MiB) of images per request, got 8,388,609 bytes", + ), +) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) +@pytest.mark.parametrize( + ("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib") +) +def test_images_over_cloudflares_size_limits_are_refused_before_http( + images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught: + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=images, + drop_params=drop_params, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + at_limits: Final = (_png_data_url(4 * _MIB), _png_object(4 * _MIB)) + + await litellm.adecisions( + model="cloudflare/clef", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": _png_data_url(4 * _MIB)}]}], + questions=_PREDICATE_QUESTIONS, + ) + await litellm.asystemone( + model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=at_limits + ) + + assert json.loads(route.calls[0].request.content)["images"] == [_png_data_url(4 * _MIB)] + assert json.loads(route.calls[1].request.content)["images"] == list(at_limits) + + +@pytest.mark.usefixtures("cloudflare_account") +def test_a_downloaded_image_over_cloudflares_limit_is_refused_before_the_cloudflare_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond( + content=bytes(4 * _MIB + 1), headers={"content-type": "image/png"} + ) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="got one of 4,194,305 bytes") as caught: + litellm.systemone( + model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_REMOTE_IMAGE_URL] + ) + + assert caught.value.status_code == 400 + assert image_route.call_count == 1 + assert not route.called + + +@pytest.mark.usefixtures("cloudflare_account") +def test_remote_image_urls_with_an_uppercase_scheme_are_inlined_for_cloudflare( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + png: Final = b"\x89PNG\r\n\x1a\n" + image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(content=png, headers={"content-type": "image/png"}) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_REMOTE_IMAGE_URL.replace("https://", "HTTPS://")], + ) + + assert image_route.call_count == 1 + assert json.loads(route.calls[0].request.content)["images"] == [ + f"data:image/png;base64,{base64.b64encode(png).decode()}" + ] + + +_PNG_DATA_URL_WITH_NAME: Final = "data:image/png;name=screen.png;base64,AA==" +_GIF_DATA_URL: Final = "data:image/gif;base64,R0lGODlh" +_CLOUDFLARE_TYPE_REFUSAL: Final = "cloudflare takes only image/jpeg, image/jpg, image/png, image/webp images, got " + + +@pytest.mark.usefixtures("openai_default_base") +def test_a_data_url_with_media_type_parameters_reaches_openai_unchanged(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=_OPENAI_RESPONSE) + + litellm.decisions( + model="openai/gpt-6-luna", + api_key="caller-key", + input=_screen_messages(_PNG_DATA_URL_WITH_NAME), + questions=_PREDICATE_QUESTIONS, + ) + + assert json.loads(route.calls[0].request.content)["input"][0]["content"][1]["image_url"] == _PNG_DATA_URL_WITH_NAME + + +@pytest.mark.usefixtures("cloudflare_account") +def test_cloudflare_gets_images_without_media_type_parameters(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + litellm.systemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[ + _PNG_DATA_URL_WITH_NAME.replace("image/png", "IMAGE/PNG"), + {"content_type": "image/webp;name=screen.webp", "base64": "BB=="}, + ], + ) + + assert json.loads(route.calls[0].request.content)["images"] == [ + _PNG_DATA_URL, + {"content_type": "image/webp", "base64": "BB=="}, + ] + + +_REFUSED_TYPE_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + ("decisions", {"input": _screen_messages(_GIF_DATA_URL), "questions": _PREDICATE_QUESTIONS}), + ( + "systemone", + { + "state": _SCREEN_TEXT, + "questions": _SYSTEMONE_PREDICATE, + "images": [{"content_type": "image/gif", "base64": "R0lGODlh"}], + }, + ), +) + + +@pytest.mark.usefixtures("cloudflare_account") +@pytest.mark.parametrize(("api", "request_kwargs"), _REFUSED_TYPE_REQUESTS, ids=("input_image", "images_object")) +def test_an_image_type_cloudflare_refuses_is_rejected_before_http( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}image/gif")) as caught: + getattr(litellm, api)(model="cloudflare/clef", **request_kwargs) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_an_image_type_cloudflare_refuses_is_dropped_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + input=( + *_screen_messages(_GIF_DATA_URL), + {"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}, + ), + questions=_PREDICATE_QUESTIONS, + drop_params=True, + ) + with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}image/gif")): + await litellm.adecisions( + model="cloudflare/clef", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": _GIF_DATA_URL}]}], + questions=_PREDICATE_QUESTIONS, + drop_params=True, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content)["images"] == [_PNG_DATA_URL] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_an_image_dropped_under_drop_params_does_not_count_toward_the_cloudflare_limit( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + gif_then_four_pngs: Final = ( + { + "role": "user", + "content": [ + {"type": "input_text", "text": _SCREEN_TEXT}, + *({"type": "input_image", "image_url": url} for url in (_GIF_DATA_URL, *(_PNG_DATA_URL,) * 4)), + ], + }, + ) + + litellm.decisions( + model="cloudflare/clef", input=gif_then_four_pngs, questions=_PREDICATE_QUESTIONS, drop_params=True + ) + await litellm.adecisions( + model="cloudflare/clef", input=gif_then_four_pngs, questions=_PREDICATE_QUESTIONS, drop_params=True + ) + + assert [json.loads(call.request.content)["images"] for call in route.calls] == [[_PNG_DATA_URL] * 4] * 2 + + +@pytest.mark.usefixtures("cloudflare_account") +def test_a_downloaded_page_that_is_not_an_image_is_refused_before_the_cloudflare_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond( + content=b"", headers={"content-type": "text/html; charset=utf-8"} + ) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}text/html")) as caught: + litellm.systemone( + model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_REMOTE_IMAGE_URL] + ) + + assert caught.value.status_code == 400 + assert image_route.call_count == 1 + assert not route.called + + @pytest.mark.asyncio @pytest.mark.parametrize( "api_base", @@ -1973,7 +2741,9 @@ async def test_clips_are_dropped_from_the_wire_for_a_model_without_media_support call: Final = getattr(litellm, f"a{api}") await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) - await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips) + await call( + model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips + ) assert len(route.calls) == 2 text_only_body: Final = json.loads(route.calls[0].request.content) @@ -2099,7 +2869,12 @@ async def test_media_support_is_read_per_kind_from_the_cost_map( ( "typesafe/jev-chatty", "https://example.com/v1/systemone", - {"litellm_provider": "typesafe", "mode": "chat", "supports_audio_input": True, "supports_video_input": True}, + { + "litellm_provider": "typesafe", + "mode": "chat", + "supports_audio_input": True, + "supports_video_input": True, + }, ), ("openai/gpt-audio", "https://example.com/v1/decisions", None), ), @@ -2121,7 +2896,9 @@ async def test_a_chat_models_media_flags_do_not_count_for_decisions( audio: Final = ["data:audio/wav;base64,YQ=="] videos: Final = ["data:video/mp4;base64,YQ=="] - with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio', 'videos']")): + with pytest.raises( + litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio', 'videos']") + ): await litellm.asystemone( model=model, state="test", diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index 57eb32f98e5..9d9aef0c22a 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -3,13 +3,11 @@ import base64 import copy import time import uuid -from unittest.mock import patch import pytest from httpx import Request, Response import litellm -from litellm import constants from litellm.litellm_core_utils.prompt_templates import image_handling from litellm.litellm_core_utils.prompt_templates.image_handling import ( MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, @@ -275,9 +273,7 @@ def _stub_image_client(content, content_type): def test_convert_url_to_base64_infers_the_type_when_the_server_sends_octet_stream(monkeypatch): - monkeypatch.setattr( - litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "application/octet-stream") - ) + monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "application/octet-stream")) result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}") @@ -285,9 +281,7 @@ def test_convert_url_to_base64_infers_the_type_when_the_server_sends_octet_strea def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch): - monkeypatch.setattr( - litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "image/jpeg") - ) + monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "image/jpeg")) result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}.png") @@ -312,7 +306,7 @@ def test_image_size_limit_disabled(monkeypatch): """ Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads. """ - import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling + from litellm.litellm_core_utils.prompt_templates import image_handling monkeypatch.setattr(litellm, "module_level_client", SmallImageClient()) monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0) @@ -580,6 +574,54 @@ def test_convert_url_to_base64_hides_the_ssrf_verdict_and_does_not_retry(monkeyp _assert_verdict_free_messages(messages, url) +def test_convert_url_to_base64_retries_within_one_deadline(monkeypatch): + timeouts = [] + + def time_out(client, url, timeout=None, **kwargs): + timeouts.append(timeout) + time.sleep(timeout) + raise litellm.Timeout(message="timed out", model="", llm_provider="") + + monkeypatch.setattr(image_handling, "safe_get", time_out) + started = time.monotonic() + with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"): + convert_url_to_base64(f"https://slow.example/{uuid.uuid4()}.png", timeout=0.3) + + assert time.monotonic() - started < 0.6 + assert len(timeouts) == 1 and 0 < timeouts[0] <= 0.3 + + +async def test_async_convert_url_to_base64_retries_within_one_deadline(monkeypatch): + timeouts = [] + + async def time_out(client, url, timeout=None, **kwargs): + timeouts.append(timeout) + await asyncio.sleep(timeout) + raise litellm.Timeout(message="timed out", model="", llm_provider="") + + monkeypatch.setattr(image_handling, "async_safe_get", time_out) + started = time.monotonic() + with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"): + await async_convert_url_to_base64(f"https://slow.example/{uuid.uuid4()}.png", timeout=0.3) + + assert time.monotonic() - started < 0.6 + assert len(timeouts) == 1 and 0 < timeouts[0] <= 0.3 + + +def test_convert_url_to_base64_still_retries_fast_failures_three_times(monkeypatch): + attempts = [] + + def refuse(client, url, timeout=None, **kwargs): + attempts.append(timeout) + raise ConnectionError("refused") + + monkeypatch.setattr(image_handling, "safe_get", refuse) + with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"): + convert_url_to_base64(f"https://down.example/{uuid.uuid4()}.png", timeout=30.0) + + assert len(attempts) == 3 and all(0 < timeout <= 30.0 for timeout in attempts) + + async def test_async_inline_remote_media_caps_in_flight_fetches_per_request(monkeypatch): in_flight = {"now": 0, "peak": 0} diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 454fa393ea8..a729ac41371 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -94,11 +94,10 @@ def test_openai_messages_become_systemone_state_text() -> None: body: Final = ir_to_systemone_request("jev-latest", ir) - assert isinstance(body, Mapping) assert body["state"] == "The package arrived broken.\n\nI want a refund.\n\nOrder 1234." -def test_image_input_is_unsupported_by_systemone_providers() -> None: +def test_openai_input_images_reach_a_systemone_provider_as_its_images_in_message_order() -> None: ir: Final = _openai_ir( { "input": [ @@ -106,15 +105,27 @@ def test_image_input_is_unsupported_by_systemone_providers() -> None: "role": "user", "content": [ {"type": "input_text", "text": "Is the screen cracked?"}, - {"type": "input_image", "image_url": "data:image/png;base64,AA=="}, + {"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "high"}, ], - } + }, + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/webp;base64,BB=="}]}, + {"role": "user", "content": "Order 1234."}, ], "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], } ) - assert isinstance(ir_to_systemone_request("jev-latest", ir), UnsupportedDecisionsRequest) + body: Final = ir_to_systemone_request("clef", ir) + + assert body["state"] == "Is the screen cracked?\n\nOrder 1234." + assert body["images"] == ["data:image/png;base64,AA==", "data:image/webp;base64,BB=="] + + +def test_systemone_images_reach_a_systemone_provider_in_the_callers_form() -> None: + images: Final = ["data:image/png;base64,AA==", {"content_type": "image/jpeg", "base64": "BB=="}] + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "images": images})) + + assert ir_to_systemone_request("clef", ir) == {"model": "clef", **_SYSTEMONE_REQUEST, "images": images} def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 1a952966514..3faae2e3289 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -673,7 +673,7 @@ def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch async def test_async_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling - async def fake_convert(url): + async def fake_convert(url, timeout=None): return f"data:image/png;base64,{url}" monkeypatch.setattr(image_handling, "async_convert_url_to_base64", fake_convert) diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index e73c5b92ab6..db781a1caf7 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -181,6 +181,32 @@ def test_systemone_questions_without_instructions_become_valid_openai_questions( assert all(question.instructions for question in _OPENAI_BODY.validate_python(body).questions) +def test_systemone_images_reach_openai_as_input_images_ahead_of_the_state() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234}, + "questions": {"damaged": {"type": "noul", "instructions": "Is the item damaged?"}}, + "images": ["data:image/png;base64,AA==", {"content_type": "image/webp", "base64": "BB=="}], + } + ) + + assert ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) == { + "model": "gpt-6-luna", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_image", "image_url": "data:image/png;base64,AA=="}, + {"type": "input_image", "image_url": "data:image/webp;base64,BB=="}, + {"type": "input_text", "text": '{"ticket": 1234}'}, + ], + } + ], + "questions": [{"type": "predicate", "name": "damaged", "instructions": "Is the item damaged?"}], + } + + @pytest.mark.parametrize( "question", ({"type": "choice", "criteria": {"refund": None}}, {"type": "score", "criteria": ["minor"]}), diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 30f0508b236..66ba28d85c1 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -520,15 +520,8 @@ def test_decisions_return_guardrail_information_when_requested( } ], }, - { - "model": "decider", - "input": [ - {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]} - ], - "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], - }, ), - ids=("systemone_body", "colliding_choice_values", "image_input"), + ids=("systemone_body", "colliding_choice_values"), ) def test_openai_format_decisions_rejects_bodies_it_cannot_translate( client: TestClient, @@ -804,6 +797,158 @@ def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_p } +_IMAGE_TEXT: Final = "Is the screen cracked?" +_PNG_DATA_URL: Final = "data:image/png;base64,AA==" +_IMAGE_ENDPOINTS: Final = ("/v1/decisions", "/v1/systemone") +_IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = ( + { + "model": "decider", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": _IMAGE_TEXT}, + {"type": "input_image", "image_url": _PNG_DATA_URL}, + ], + } + ], + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + }, + { + "model": "decider", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "images": [_PNG_DATA_URL], + }, +) +_IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True)) +_IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format") + + +def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router(model_list=[{"model_name": "decider", "litellm_params": dict(litellm_params)}]), + ) + + +@pytest.mark.parametrize(("endpoint", "request_body"), _IMAGE_REQUESTS, ids=_IMAGE_REQUEST_IDS) +def test_images_reach_a_cloudflare_clef_deployment_as_its_images_field( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], +) -> None: + _route_decider_to( + monkeypatch, + { + "model": "cloudflare/clef", + "api_key": "cloudflare-key", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + }, + ) + upstream: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=request_body) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "images": [_PNG_DATA_URL], + } + + +@pytest.mark.parametrize( + ("endpoint", "request_body", "param"), + tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, ("input_image", "images"), strict=True)), + ids=_IMAGE_REQUEST_IDS, +) +def test_images_to_a_text_only_deployment_are_refused_without_drop_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], + param: str, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=request_body) + + assert response.status_code == 400, response.text + assert f"['{param}']" in response.json()["error"]["message"] + assert not upstream.called + + +@pytest.mark.parametrize(("endpoint", "request_body"), _IMAGE_REQUESTS, ids=_IMAGE_REQUEST_IDS) +def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_decider_to( + monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True} + ) + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=request_body) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_malformed_images_are_refused_unless_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_decider_to( + monkeypatch, + { + "model": "cloudflare/clef", + "api_key": "cloudflare-key", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + "drop_params": deployment_drops_params, + }, + ) + upstream: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/systemone", json={**_IMAGE_REQUEST_BODIES[1], "images": [{"url": _PNG_DATA_URL}]} + ) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "images" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + @pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) @pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) def test_clips_for_a_model_without_media_support_are_refused_unless_the_deployment_drops_params( diff --git a/tests/unit/systemone/test_main.py b/tests/unit/systemone/test_main.py index 342b7f1ef97..8b7537308c1 100644 --- a/tests/unit/systemone/test_main.py +++ b/tests/unit/systemone/test_main.py @@ -28,6 +28,9 @@ _SYSTEMONE_RESPONSE: Final[Mapping[str, object]] = { "usage": {"input_tokens": 12, "output_tokens": 1}, } _TYPESAFE_URL: Final = "https://api.typesafe.ai/v1/systemone" +_CLOUDFLARE_API_BASE: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run" +_CLOUDFLARE_CLEF_URL: Final = f"{_CLOUDFLARE_API_BASE}/@cf/cloudflare/clef" +_PNG_DATA_URL: Final = "data:image/png;base64,AA==" @pytest.fixture(autouse=True) @@ -105,6 +108,25 @@ async def test_decisions_with_state_still_sends_the_systemone_format_and_warns( assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} +@pytest.mark.asyncio +async def test_decisions_with_state_still_sends_its_images(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_CLOUDFLARE_CLEF_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.warns(UserWarning, match=r"litellm\.systemone"): + await litellm.adecisions( + model="cloudflare/clef", + state=_STATE, + questions=_SYSTEMONE_QUESTIONS, + images=[_PNG_DATA_URL], + api_key="caller-key", + api_base=_CLOUDFLARE_API_BASE, + ) + + sent: Final = json.loads(respx_mock.calls.last.request.content) + assert sent["images"] == [_PNG_DATA_URL] + assert route.called + + @pytest.mark.asyncio async def test_systemone_rejects_openai_question_lists_before_http(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2cc91879f7e..b5b164e62cc 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -71553,6 +71553,22 @@ export interface operations { */ content_type: string; })[]; + /** + * Images + * @description Images as data URLs or content_type plus base64, for providers that look at images + */ + images?: (string | { + /** + * Base64 + * @description Base64-encoded image bytes + */ + base64: string; + /** + * Content Type + * @description An image/* media type, such as image/png + */ + content_type: string; + })[] | null; /** * Model * @description A decision model from the proxy model_list @@ -80417,6 +80433,22 @@ export interface operations { */ content_type: string; })[]; + /** + * Images + * @description Images as data URLs or content_type plus base64, for providers that look at images + */ + images?: (string | { + /** + * Base64 + * @description Base64-encoded image bytes + */ + base64: string; + /** + * Content Type + * @description An image/* media type, such as image/png + */ + content_type: string; + })[] | null; /** * Model * @description A decision model from the proxy model_list