mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
feat(decisions)!: send images to Cloudflare Clef through its images field (#45734)
* feat(decisions): send images to Cloudflare Clef OpenAI-format input_image parts and the System One images field now reach Cloudflare clef and clef-flash in its images field, with remote URLs downloaded and inlined first. Providers that cannot take images drop them under drop_params and answer 400 otherwise. * fix(decisions): follow drop_params for images and use the call timeout for image downloads * fix(decisions): drop a malformed images entry on its own under drop_params * fix(decisions): refuse images next to input whatever drop_params says * fix(decisions): bound clef image fetches, honor drop_params for images, and log caller URLs Refuse more than 4 images for Cloudflare clef before any download, keep the caller's timeout as one deadline across image fetch retries, accept images next to input under drop_params, validate image strings and objects, and pass the caller's image URLs (not inlined bytes) to logging callbacks. * fix(decisions): check input_image URLs and clef image sizes * fix(decisions): accept data URL media type parameters and check clef image types * fix(decisions): count clef images after dropping unsupported types and flag clef vision --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
30b8db325e
commit
2f3ccbe7c7
24 changed files with 2968 additions and 144 deletions
|
|
@ -1,5 +1,7 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
|
|
@ -9,19 +11,33 @@ from typing_extensions import assert_never
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_urls_to_base64,
|
||||
convert_url_to_base64,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
ir_to_systemone_response,
|
||||
remote_image_urls,
|
||||
systemone_request_to_ir,
|
||||
with_images_where,
|
||||
with_inlined_images,
|
||||
)
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
|
||||
from litellm.types.decisions import (
|
||||
DecisionQuestion,
|
||||
DecisionsAudio,
|
||||
DecisionsBase64Image,
|
||||
DecisionsImage,
|
||||
DecisionsIRMessages,
|
||||
DecisionsIRRequest,
|
||||
DecisionsIRResponse,
|
||||
DecisionsIRState,
|
||||
DecisionsJSON,
|
||||
DecisionsRequestBody,
|
||||
DecisionsResponse,
|
||||
|
|
@ -37,6 +53,7 @@ from litellm.utils import ProviderConfigManager
|
|||
|
||||
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
|
||||
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
|
||||
DecisionsImages: TypeAlias = Sequence[str | DecisionsBase64Image | Mapping[str, object]]
|
||||
DecisionsAudioClips: TypeAlias = Sequence[DecisionsAudio | Mapping[str, object]]
|
||||
DecisionsVideoClips: TypeAlias = Sequence[DecisionsVideo | Mapping[str, object]]
|
||||
DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
|
||||
|
|
@ -46,19 +63,62 @@ _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAda
|
|||
_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(
|
||||
str | None, config=ConfigDict(title="safety_identifier")
|
||||
)
|
||||
_IMAGES_ADAPTER: Final[TypeAdapter[tuple[DecisionsImage, ...] | None]] = TypeAdapter(
|
||||
tuple[DecisionsImage, ...] | None, config=ConfigDict(title="images")
|
||||
)
|
||||
_HANDLER: Final = BaseLLMHTTPHandler()
|
||||
_BASE64_MARKER: Final = ";base64,"
|
||||
|
||||
|
||||
_IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images"))
|
||||
_IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _valid_image(image: object) -> DecisionsImage | None:
|
||||
try:
|
||||
return _IMAGE_ADAPTER.validate_python(image)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _image_entries(images: object) -> tuple[object, ...]:
|
||||
try:
|
||||
return _IMAGE_ENTRIES_ADAPTER.validate_python(images)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _is_well_formed_image(image: DecisionsImage) -> bool:
|
||||
return _valid_image(image) is not None
|
||||
|
||||
|
||||
def _no_image(_image: DecisionsImage) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]:
|
||||
if not drops_params:
|
||||
return _IMAGES_ADAPTER.validate_python(images) or ()
|
||||
return tuple(image for image in map(_valid_image, _image_entries(images)) if image is not None)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SystemOneRequestFields:
|
||||
state: DecisionsJSON | None
|
||||
questions: SystemOneQuestions | None
|
||||
images: object = None
|
||||
audio: DecisionsAudioClips | None = None
|
||||
videos: DecisionsVideoClips | None = None
|
||||
|
||||
def validate(self, safety_identifier: str | None) -> DecisionsRequestBody:
|
||||
def validate(self, *, safety_identifier: str | None, drops_params: bool) -> DecisionsRequestBody:
|
||||
return _SYSTEMONE_REQUEST_ADAPTER.validate_python(
|
||||
{"state": self.state, "questions": self.questions, "audio": self.audio, "videos": self.videos}
|
||||
{
|
||||
"state": self.state,
|
||||
"questions": self.questions,
|
||||
"images": _request_images(self.images, drops_params) or None,
|
||||
"audio": self.audio,
|
||||
"videos": self.videos,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -69,7 +129,7 @@ class OpenAIDecisionRequestFields:
|
|||
audio: DecisionsAudioClips | None = None
|
||||
videos: DecisionsVideoClips | None = None
|
||||
|
||||
def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody:
|
||||
def validate(self, *, safety_identifier: str | None, drops_params: bool) -> OpenAIDecisionRequestBody:
|
||||
return _OPENAI_REQUEST_ADAPTER.validate_python(
|
||||
{
|
||||
"input": self.input,
|
||||
|
|
@ -92,7 +152,6 @@ class DecisionsCall:
|
|||
provider_config: BaseDecisionsConfig
|
||||
request: DecisionsRequestFormat
|
||||
ir_request: DecisionsIRRequest
|
||||
body: Mapping[str, object] = field(repr=False)
|
||||
api_base: str
|
||||
api_key: str | None = field(repr=False)
|
||||
logging_obj: LiteLLMLoggingObj | None
|
||||
|
|
@ -169,18 +228,61 @@ class _UnsupportedParams:
|
|||
names: tuple[str, ...]
|
||||
|
||||
|
||||
def _unsupported_param_names(
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _NoInputLeftAfterDrop:
|
||||
names: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _MalformedImages:
|
||||
count: int
|
||||
|
||||
|
||||
def _unsupported_params(
|
||||
ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str
|
||||
) -> tuple[str, ...]:
|
||||
checks: Final = (
|
||||
(
|
||||
"safety_identifier",
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier,
|
||||
),
|
||||
("audio", bool(ir_request.audio) and not provider_config.supports_audio_input(model, custom_llm_provider)),
|
||||
("videos", bool(ir_request.videos) and not provider_config.supports_video_input(model, custom_llm_provider)),
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
)
|
||||
return tuple(name for name, unsupported in checks if unsupported)
|
||||
refused_image_param: Final = None if provider_config.supports_images else image_param(ir_request.input)
|
||||
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
return (
|
||||
*(("safety_identifier",) if refuses_safety_identifier else ()),
|
||||
*(() if refused_image_param is None else (refused_image_param,)),
|
||||
*(("audio",) if refuses_audio else ()),
|
||||
*(("videos",) if refuses_videos else ()),
|
||||
)
|
||||
|
||||
|
||||
def _without_unsupported_params(
|
||||
ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
input=ir_request.input if provider_config.supports_images else with_images_where(ir_request.input, _no_image),
|
||||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
)
|
||||
|
||||
|
||||
def _leaves_no_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> bool:
|
||||
return isinstance(decision_input, DecisionsIRMessages) and not decision_input.messages
|
||||
|
||||
|
||||
def _without_malformed_images(
|
||||
ir_request: DecisionsIRRequest, drops_params: bool
|
||||
) -> DecisionsIRRequest | _MalformedImages:
|
||||
malformed: Final = sum(1 for image in input_images(ir_request.input) if not _is_well_formed_image(image))
|
||||
if not malformed:
|
||||
return ir_request
|
||||
well_formed: Final = replace(ir_request, input=with_images_where(ir_request.input, _is_well_formed_image))
|
||||
if not drops_params or _leaves_no_input(well_formed.input):
|
||||
return _MalformedImages(count=malformed)
|
||||
return well_formed
|
||||
|
||||
|
||||
def _provider_ir_request(
|
||||
|
|
@ -190,12 +292,20 @@ def _provider_ir_request(
|
|||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
) -> DecisionsIRRequest | _UnsupportedParams:
|
||||
ir_request: Final = _ir_request(request, safety_identifier)
|
||||
unsupported: Final = _unsupported_param_names(ir_request, provider_config, model, custom_llm_provider)
|
||||
if unsupported:
|
||||
return _UnsupportedParams(unsupported)
|
||||
return ir_request
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionsIRRequest | _UnsupportedParams | _NoInputLeftAfterDrop | _MalformedImages:
|
||||
ir_request: Final = _without_malformed_images(_ir_request(request, safety_identifier), _drops_params(kwargs))
|
||||
if isinstance(ir_request, _MalformedImages):
|
||||
return ir_request
|
||||
unsupported: Final = _unsupported_params(ir_request, provider_config, model, custom_llm_provider)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NoInputLeftAfterDrop(names=unsupported)
|
||||
return supported
|
||||
|
||||
|
||||
def prepare_call(
|
||||
|
|
@ -231,7 +341,7 @@ def prepare_call(
|
|||
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
|
||||
request: Final = _fields_without_unsupported_media(
|
||||
fields, provider_config, canonical_model, provider, kwargs
|
||||
).validate(request_safety_identifier)
|
||||
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
|
|
@ -260,6 +370,7 @@ def prepare_call(
|
|||
provider_config=provider_config,
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if isinstance(ir_request, _UnsupportedParams):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
|
|
@ -271,16 +382,24 @@ def prepare_call(
|
|||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
body: Final = provider_config.transform_decisions_request(
|
||||
model=canonical_model, request=ir_request, custom_llm_provider=provider
|
||||
)
|
||||
if isinstance(body, UnsupportedDecisionsRequest):
|
||||
if isinstance(ir_request, _MalformedImages):
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}",
|
||||
message=(
|
||||
f"Invalid Decisions request: {ir_request.count} input_image part(s) have an image_url that is "
|
||||
"neither a data:image/...;base64, URL nor an http(s) URL"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
if isinstance(ir_request, _NoInputLeftAfterDrop):
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, "
|
||||
"and dropping them leaves no input to decide on"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.update_from_kwargs(
|
||||
|
|
@ -299,7 +418,6 @@ def prepare_call(
|
|||
provider_config=provider_config,
|
||||
request=request,
|
||||
ir_request=ir_request,
|
||||
body=body,
|
||||
api_base=resolved_api_base,
|
||||
api_key=resolved_api_key,
|
||||
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
|
||||
|
|
@ -309,6 +427,122 @@ def prepare_call(
|
|||
)
|
||||
|
||||
|
||||
def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]:
|
||||
if not call.provider_config.inlines_remote_images:
|
||||
return ()
|
||||
return remote_image_urls(call.ir_request.input)
|
||||
|
||||
|
||||
def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall:
|
||||
return replace(
|
||||
call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls))
|
||||
)
|
||||
|
||||
|
||||
def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
|
||||
remote_urls: Final = _remote_image_urls(call)
|
||||
if not remote_urls:
|
||||
return call
|
||||
return _with_data_urls(
|
||||
call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls})
|
||||
)
|
||||
|
||||
|
||||
async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
|
||||
remote_urls: Final = _remote_image_urls(call)
|
||||
if not remote_urls:
|
||||
return call
|
||||
data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
|
||||
return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
|
||||
|
||||
|
||||
def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool:
|
||||
media_type: Final = image_media_type(image)
|
||||
return media_type is None or media_type in media_types
|
||||
|
||||
|
||||
def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
|
||||
media_types: Final = call.provider_config.image_media_types
|
||||
if media_types is None:
|
||||
return call
|
||||
found: Final = (image_media_type(image) for image in input_images(call.ir_request.input))
|
||||
unsupported: Final = tuple(
|
||||
dict.fromkeys(type_ for type_ in found if type_ is not None and type_ not in media_types)
|
||||
)
|
||||
if not unsupported:
|
||||
return call
|
||||
supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(supported):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=supported))
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, "
|
||||
f"got {', '.join(unsupported)}"
|
||||
),
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def _checked_image_count(call: DecisionsCall) -> DecisionsCall:
|
||||
max_images: Final = call.provider_config.max_images
|
||||
image_count: Final = len(input_images(call.ir_request.input))
|
||||
if max_images is None or image_count <= max_images:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def _decoded_image_bytes(image: DecisionsImage) -> int:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return len(image.base64.rstrip("=")) * 3 // 4
|
||||
marker_at: Final = image.lower().find(_BASE64_MARKER)
|
||||
return 0 if marker_at < 0 else len(image[marker_at + len(_BASE64_MARKER) :].rstrip("=")) * 3 // 4
|
||||
|
||||
|
||||
def _byte_limit(limit: int) -> str:
|
||||
return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)"
|
||||
|
||||
|
||||
def _image_size_error(call: DecisionsCall) -> str | None:
|
||||
sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
largest: Final = max(sizes, default=0)
|
||||
per_image: Final = call.provider_config.max_image_bytes
|
||||
total: Final = call.provider_config.max_total_image_bytes
|
||||
if per_image is not None and largest > per_image:
|
||||
return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes"
|
||||
if total is not None and sum(sizes) > total:
|
||||
return f"takes at most {_byte_limit(total)} of images per request, got {sum(sizes):,} bytes"
|
||||
return None
|
||||
|
||||
|
||||
def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall:
|
||||
error: Final = _image_size_error(call)
|
||||
if error is None:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} {error}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
||||
body: Final = call.provider_config.transform_decisions_request(
|
||||
model=call.model, request=call.ir_request, custom_llm_provider=call.custom_llm_provider
|
||||
)
|
||||
if isinstance(body, UnsupportedDecisionsRequest):
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Decisions provider '{call.custom_llm_provider}' cannot serve this request: {body.reason}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
return body
|
||||
|
||||
|
||||
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
formatted: Final = _formatted_response(response, call)
|
||||
provider_cost: Final = call.provider_config.provider_reported_cost(response)
|
||||
|
|
@ -351,43 +585,51 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
|
|||
|
||||
|
||||
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
supported: Final = _checked_image_count(_checked_image_types(call))
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported)))
|
||||
body: Final = _request_body(inlined)
|
||||
try:
|
||||
response: Final = await _HANDLER.adecisions(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
request=call.ir_request,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
model=inlined.model,
|
||||
custom_llm_provider=inlined.custom_llm_provider,
|
||||
logging_obj=inlined.logging_obj,
|
||||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
timeout=inlined.timeout,
|
||||
litellm_params=inlined.litellm_params,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, call) from error
|
||||
return _format_response(response, call)
|
||||
raise _map_upstream_exception(error, inlined) from error
|
||||
return _format_response(response, inlined)
|
||||
|
||||
|
||||
def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
supported: Final = _checked_image_count(_checked_image_types(call))
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported)))
|
||||
body: Final = _request_body(inlined)
|
||||
try:
|
||||
response: Final = _HANDLER.decisions(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
request=call.ir_request,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
model=inlined.model,
|
||||
custom_llm_provider=inlined.custom_llm_provider,
|
||||
logging_obj=inlined.logging_obj,
|
||||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
timeout=inlined.timeout,
|
||||
litellm_params=inlined.litellm_params,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, call) from error
|
||||
return _format_response(response, call)
|
||||
raise _map_upstream_exception(error, inlined) from error
|
||||
return _format_response(response, inlined)
|
||||
|
||||
|
||||
def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None:
|
||||
|
|
@ -398,3 +640,16 @@ def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name:
|
|||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
||||
|
||||
def reject_images_next_to_input(*, model: str, kwargs: Mapping[str, object]) -> None:
|
||||
if not kwargs.get("images") or _drops_params(kwargs):
|
||||
return
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
"images belongs to the System One format; with input (OpenAI format), "
|
||||
"send images as input_image content parts"
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.decisions.call import (
|
|||
SystemOneRequestFields,
|
||||
asend,
|
||||
prepare_call,
|
||||
reject_images_next_to_input,
|
||||
send,
|
||||
)
|
||||
from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse
|
||||
|
|
@ -26,9 +27,11 @@ _SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = Ty
|
|||
_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter(
|
||||
OpenAIDecisionQuestions | None
|
||||
)
|
||||
_SYSTEMONE_FIELDS: Final = frozenset({"state", "images"})
|
||||
|
||||
|
||||
def _request_fields(
|
||||
model: str,
|
||||
decision_input: OpenAIDecisionInput | None,
|
||||
questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
|
||||
audio: DecisionsAudioClips | None,
|
||||
|
|
@ -36,6 +39,7 @@ def _request_fields(
|
|||
kwargs: Mapping[str, object],
|
||||
) -> RequestFields:
|
||||
if "state" not in kwargs:
|
||||
reject_images_next_to_input(model=model, kwargs=kwargs)
|
||||
return OpenAIDecisionRequestFields(
|
||||
input=decision_input,
|
||||
questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions),
|
||||
|
|
@ -50,6 +54,7 @@ def _request_fields(
|
|||
return SystemOneRequestFields(
|
||||
state=_STATE_ADAPTER.validate_python(kwargs["state"]),
|
||||
questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions),
|
||||
images=kwargs.get("images"),
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
|
|
@ -72,14 +77,14 @@ def _prepare_decisions_call(
|
|||
) -> DecisionsCall:
|
||||
return prepare_call(
|
||||
model=model,
|
||||
fields=_request_fields(decision_input, questions, audio, videos, kwargs),
|
||||
fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),
|
||||
safety_identifier=safety_identifier,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs={key: value for key, value in kwargs.items() if key != "state"},
|
||||
kwargs={key: value for key, value in kwargs.items() if key not in _SYSTEMONE_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@ Helper functions to handle images passed in messages
|
|||
|
||||
import asyncio
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
|
||||
import litellm
|
||||
|
|
@ -22,6 +24,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
|
||||
MAX_IMGS_IN_MEMORY: Final = 10
|
||||
MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
|
||||
_FETCH_ATTEMPTS: Final = 3
|
||||
|
||||
in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
|
||||
|
||||
|
|
@ -73,6 +76,13 @@ def _process_image_response(response: Response, url: str) -> str:
|
|||
return result
|
||||
|
||||
|
||||
def _attempt_timeouts(timeout: float | httpx.Timeout | None) -> Iterable[float | httpx.Timeout | None]:
|
||||
if timeout is None or isinstance(timeout, httpx.Timeout):
|
||||
return (timeout,) * _FETCH_ATTEMPTS
|
||||
deadline: Final = time.monotonic() + timeout
|
||||
return (remaining for _ in range(_FETCH_ATTEMPTS) if (remaining := deadline - time.monotonic()) > 0)
|
||||
|
||||
|
||||
def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchError":
|
||||
verbose_logger.warning("Image fetch of %s rejected before any request went out: %s", url, verdict)
|
||||
return litellm.ImageFetchError(
|
||||
|
|
@ -81,7 +91,7 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr
|
|||
)
|
||||
|
||||
|
||||
async def async_convert_url_to_base64(url: str) -> str:
|
||||
async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
|
||||
if url.startswith("data:") and ";base64," in url:
|
||||
return url
|
||||
|
||||
|
|
@ -96,9 +106,9 @@ async def async_convert_url_to_base64(url: str) -> str:
|
|||
return cached_result
|
||||
|
||||
client: Final = litellm.module_level_aclient
|
||||
for _ in range(3):
|
||||
for attempt_timeout in _attempt_timeouts(timeout):
|
||||
try:
|
||||
response = await async_safe_get(client, url)
|
||||
response = await async_safe_get(client, url, timeout=attempt_timeout)
|
||||
return _process_image_response(response, url)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
|
|
@ -109,7 +119,7 @@ async def async_convert_url_to_base64(url: str) -> str:
|
|||
raise litellm.ImageFetchError(f"Error: Unable to fetch image from URL after 3 attempts. url={url}")
|
||||
|
||||
|
||||
def convert_url_to_base64(url: str) -> str:
|
||||
def convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
|
||||
if url.startswith("data:") and ";base64," in url:
|
||||
return url
|
||||
|
||||
|
|
@ -124,9 +134,9 @@ def convert_url_to_base64(url: str) -> str:
|
|||
return cached_result
|
||||
|
||||
client: Final = litellm.module_level_client
|
||||
for _ in range(3):
|
||||
for attempt_timeout in _attempt_timeouts(timeout):
|
||||
try:
|
||||
response = safe_get(client, url)
|
||||
response = safe_get(client, url, timeout=attempt_timeout)
|
||||
return _process_image_response(response, url)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
|
|
@ -286,14 +296,16 @@ def _inline_message(
|
|||
return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined
|
||||
|
||||
|
||||
async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore) -> str:
|
||||
async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
|
||||
async with in_flight:
|
||||
return await async_convert_url_to_base64(url)
|
||||
return await async_convert_url_to_base64(url, timeout=timeout)
|
||||
|
||||
|
||||
async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]:
|
||||
async def async_convert_urls_to_base64(
|
||||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
|
||||
) -> tuple[str, ...]:
|
||||
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
|
||||
fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight)) for url in remote_urls)
|
||||
fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
|
||||
try:
|
||||
return tuple(await asyncio.gather(*fetches))
|
||||
except BaseException:
|
||||
|
|
@ -329,6 +341,6 @@ async def async_inline_remote_media(
|
|||
remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
|
||||
if not remote_urls:
|
||||
return messages
|
||||
data_urls: Final = await _fetch_data_urls(remote_urls)
|
||||
data_urls: Final = await async_convert_urls_to_base64(remote_urls)
|
||||
inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
|
||||
return [_inline_message(message, inlined, should_inline) for message in messages]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from abc import ABC
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -16,6 +18,8 @@ from litellm.types.decisions import (
|
|||
ChoiceQuestion,
|
||||
DecisionAnswer,
|
||||
DecisionQuestion,
|
||||
DecisionsBase64Image,
|
||||
DecisionsImage,
|
||||
DecisionsIRAnswer,
|
||||
DecisionsIRChoiceAnswer,
|
||||
DecisionsIRChoiceOption,
|
||||
|
|
@ -53,9 +57,8 @@ from litellm.types.utils import is_decisions_model_mode
|
|||
_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
_SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
|
||||
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
|
||||
_TEXT_ONLY: Final = UnsupportedDecisionsRequest(
|
||||
reason="input_image content parts are not supported because System One providers accept text input only"
|
||||
)
|
||||
_REMOTE_URL_PREFIXES: Final = ("http://", "https://")
|
||||
_DATA_URL_PREFIX: Final = "data:"
|
||||
|
||||
|
||||
def decisions_text(value: DecisionsJSON) -> str:
|
||||
|
|
@ -101,39 +104,153 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion:
|
|||
|
||||
def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest:
|
||||
return DecisionsIRRequest(
|
||||
input=DecisionsIRState(state=request.state),
|
||||
input=DecisionsIRState(state=request.state, images=tuple(request.images or ())),
|
||||
audio=request.audio,
|
||||
videos=request.videos,
|
||||
questions=tuple(_ir_question(name, question) for name, question in request.questions.items()),
|
||||
)
|
||||
|
||||
|
||||
def _has_image(message: OpenAIDecisionInputMessage) -> bool:
|
||||
return not isinstance(message.content, str) and any(
|
||||
isinstance(part, OpenAIDecisionInputImage) for part in message.content
|
||||
def _image_urls(message: OpenAIDecisionInputMessage) -> tuple[str, ...]:
|
||||
if isinstance(message.content, str):
|
||||
return ()
|
||||
return tuple(part.image_url for part in message.content if isinstance(part, OpenAIDecisionInputImage))
|
||||
|
||||
|
||||
def input_images(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[DecisionsImage, ...]:
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return decision_input.images
|
||||
case DecisionsIRMessages():
|
||||
return tuple(chain.from_iterable(_image_urls(message) for message in decision_input.messages))
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def image_param(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | None:
|
||||
if not input_images(decision_input):
|
||||
return None
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return "images"
|
||||
case DecisionsIRMessages():
|
||||
return "input_image"
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def _kept_part(part: OpenAIDecisionInputText | OpenAIDecisionInputImage, keep: Callable[[str], bool]) -> bool:
|
||||
return isinstance(part, OpenAIDecisionInputText) or keep(part.image_url)
|
||||
|
||||
|
||||
def _message_with_images_where(
|
||||
message: OpenAIDecisionInputMessage, keep: Callable[[str], bool]
|
||||
) -> OpenAIDecisionInputMessage | None:
|
||||
if isinstance(message.content, str):
|
||||
return message
|
||||
kept_parts: Final = tuple(part for part in message.content if _kept_part(part, keep))
|
||||
return message.model_copy(update={"content": kept_parts}) if kept_parts else None
|
||||
|
||||
|
||||
def with_images_where(
|
||||
decision_input: DecisionsIRState | DecisionsIRMessages, keep: Callable[[DecisionsImage], bool]
|
||||
) -> DecisionsIRState | DecisionsIRMessages:
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return replace(decision_input, images=tuple(image for image in decision_input.images if keep(image)))
|
||||
case DecisionsIRMessages():
|
||||
return DecisionsIRMessages(
|
||||
messages=tuple(
|
||||
kept
|
||||
for message in decision_input.messages
|
||||
if (kept := _message_with_images_where(message, keep)) is not None
|
||||
)
|
||||
)
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def remote_image_urls(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
image
|
||||
for image in input_images(decision_input)
|
||||
if isinstance(image, str) and image.lower().startswith(_REMOTE_URL_PREFIXES)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _message_text(message: OpenAIDecisionInputMessage) -> str:
|
||||
def _inlined_image(image: DecisionsImage, data_urls: Mapping[str, str]) -> DecisionsImage:
|
||||
return data_urls.get(image, image) if isinstance(image, str) else image
|
||||
|
||||
|
||||
def _inlined_part(
|
||||
part: OpenAIDecisionInputText | OpenAIDecisionInputImage, data_urls: Mapping[str, str]
|
||||
) -> OpenAIDecisionInputText | OpenAIDecisionInputImage:
|
||||
if isinstance(part, OpenAIDecisionInputImage) and part.image_url in data_urls:
|
||||
return part.model_copy(update={"image_url": data_urls[part.image_url]})
|
||||
return part
|
||||
|
||||
|
||||
def _inlined_message(message: OpenAIDecisionInputMessage, data_urls: Mapping[str, str]) -> OpenAIDecisionInputMessage:
|
||||
if isinstance(message.content, str):
|
||||
return message.content
|
||||
return "\n\n".join(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText))
|
||||
return message
|
||||
return message.model_copy(update={"content": tuple(_inlined_part(part, data_urls) for part in message.content)})
|
||||
|
||||
|
||||
def _systemone_state(
|
||||
decision_input: DecisionsIRState | DecisionsIRMessages,
|
||||
) -> DecisionsJSON | UnsupportedDecisionsRequest:
|
||||
def with_inlined_images(
|
||||
decision_input: DecisionsIRState | DecisionsIRMessages, data_urls: Mapping[str, str]
|
||||
) -> DecisionsIRState | DecisionsIRMessages:
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return replace(
|
||||
decision_input, images=tuple(_inlined_image(image, data_urls) for image in decision_input.images)
|
||||
)
|
||||
case DecisionsIRMessages():
|
||||
return DecisionsIRMessages(
|
||||
messages=tuple(_inlined_message(message, data_urls) for message in decision_input.messages)
|
||||
)
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def _message_texts(message: OpenAIDecisionInputMessage) -> tuple[str, ...]:
|
||||
if isinstance(message.content, str):
|
||||
return (message.content,)
|
||||
return tuple(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText))
|
||||
|
||||
|
||||
def _systemone_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> DecisionsJSON:
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return decision_input.state
|
||||
case DecisionsIRMessages():
|
||||
if any(_has_image(message) for message in decision_input.messages):
|
||||
return _TEXT_ONLY
|
||||
return "\n\n".join(_message_text(message) for message in decision_input.messages)
|
||||
return "\n\n".join(chain.from_iterable(_message_texts(message) for message in decision_input.messages))
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def _media_type(content_type: str) -> str:
|
||||
return content_type.split(";", 1)[0].strip().lower()
|
||||
|
||||
|
||||
def image_media_type(image: DecisionsImage) -> str | None:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return _media_type(image.content_type)
|
||||
if not image.lower().startswith(_DATA_URL_PREFIX):
|
||||
return None
|
||||
return _media_type(image[len(_DATA_URL_PREFIX) :])
|
||||
|
||||
|
||||
def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return {"content_type": image_media_type(image), "base64": image.base64}
|
||||
media_type: Final = image_media_type(image)
|
||||
if media_type is None:
|
||||
return image
|
||||
return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}"
|
||||
|
||||
|
||||
def _optional(key: str, value: object) -> Mapping[str, object]:
|
||||
return {} if value is None else {key: value}
|
||||
|
||||
|
|
@ -177,9 +294,8 @@ def ir_to_systemone_request(
|
|||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
state: Final = _systemone_state(request.input)
|
||||
if isinstance(state, UnsupportedDecisionsRequest):
|
||||
return state
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
return {
|
||||
"model": model,
|
||||
"state": state,
|
||||
|
|
@ -194,6 +310,7 @@ def ir_to_systemone_request(
|
|||
else {}
|
||||
),
|
||||
"questions": {key: _systemone_question(question) for key, question in keyed_questions},
|
||||
**({"images": [_systemone_image(image) for image in images]} if images else {}),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -328,6 +445,12 @@ class BaseDecisionsConfig(ABC):
|
|||
api_base_env: tuple[str, ...] = ()
|
||||
api_key_required: bool = True
|
||||
supports_safety_identifier: bool = False
|
||||
supports_images: bool = False
|
||||
max_images: int | None = None
|
||||
max_image_bytes: int | None = None
|
||||
max_total_image_bytes: int | None = None
|
||||
image_media_types: frozenset[str] | None = None
|
||||
inlines_remote_images: bool = False
|
||||
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
|
||||
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,12 @@ _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapte
|
|||
class CloudflareDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("CLOUDFLARE_API_KEY",)
|
||||
api_base_env = ("CLOUDFLARE_API_BASE",)
|
||||
supports_images = True
|
||||
max_images = 4
|
||||
max_image_bytes = 4 * 1024 * 1024
|
||||
max_total_image_bytes = 8 * 1024 * 1024
|
||||
image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"})
|
||||
inlines_remote_images = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
|
||||
|
|
|
|||
|
|
@ -1489,6 +1489,7 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
body: Mapping[str, object],
|
||||
logged_body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
|
|
@ -1498,7 +1499,7 @@ class BaseLLMHTTPHandler:
|
|||
data: Final = dict(body)
|
||||
if logging_obj is not None:
|
||||
logging_obj.pre_call(
|
||||
input=data,
|
||||
input=dict(logged_body),
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers},
|
||||
|
|
@ -1513,6 +1514,7 @@ class BaseLLMHTTPHandler:
|
|||
provider_config: BaseDecisionsConfig,
|
||||
request: DecisionsIRRequest,
|
||||
body: Mapping[str, object],
|
||||
logged_body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
|
|
@ -1525,6 +1527,7 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
body=body,
|
||||
logged_body=logged_body,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
|
|
@ -1561,6 +1564,7 @@ class BaseLLMHTTPHandler:
|
|||
provider_config: BaseDecisionsConfig,
|
||||
request: DecisionsIRRequest,
|
||||
body: Mapping[str, object],
|
||||
logged_body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
|
|
@ -1573,6 +1577,7 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
body=body,
|
||||
logged_body=logged_body,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import litellm
|
|||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.decisions import (
|
||||
DecisionsImage,
|
||||
DecisionsIRAnswer,
|
||||
DecisionsIRChoiceAnswer,
|
||||
DecisionsIRChoiceOption,
|
||||
|
|
@ -32,6 +33,9 @@ from litellm.types.decisions import (
|
|||
OpenAIChoiceQuestion,
|
||||
OpenAIDecisionAnswer,
|
||||
OpenAIDecisionInput,
|
||||
OpenAIDecisionInputImage,
|
||||
OpenAIDecisionInputMessage,
|
||||
OpenAIDecisionInputText,
|
||||
OpenAIDecisionInputTokensDetails,
|
||||
OpenAIDecisionOutputTokensDetails,
|
||||
OpenAIDecisionQuestion,
|
||||
|
|
@ -145,8 +149,23 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
assert_never(question)
|
||||
|
||||
|
||||
def _image_url(image: DecisionsImage) -> str:
|
||||
return image if isinstance(image, str) else f"data:{image.content_type};base64,{image.base64}"
|
||||
|
||||
|
||||
def _state_message(state: DecisionsIRState) -> OpenAIDecisionInputMessage:
|
||||
return OpenAIDecisionInputMessage(
|
||||
content=(
|
||||
*(OpenAIDecisionInputImage(type="input_image", image_url=_image_url(image)) for image in state.images),
|
||||
OpenAIDecisionInputText(type="input_text", text=decisions_text(state.state)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | Sequence[Mapping[str, object]]:
|
||||
match decision_input:
|
||||
case DecisionsIRState() if decision_input.images:
|
||||
return [_state_message(decision_input).model_dump(mode="json", exclude_none=True)]
|
||||
case DecisionsIRState():
|
||||
return decisions_text(decision_input.state)
|
||||
case DecisionsIRMessages():
|
||||
|
|
@ -297,6 +316,7 @@ def ir_to_openai_response(
|
|||
class OpenAIDecisionsConfig(BaseDecisionsConfig):
|
||||
path = "/v1/decisions"
|
||||
supports_safety_identifier = True
|
||||
supports_images = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://api.openai.com"
|
||||
|
|
|
|||
|
|
@ -16029,7 +16029,8 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
|
||||
"supports_audio_input": true,
|
||||
"supports_video_input": true
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-omni": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
|
|
@ -16039,7 +16040,8 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
|
||||
"supports_audio_input": true,
|
||||
"supports_video_input": true
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
|
|
@ -16047,7 +16049,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
|
|
@ -16055,7 +16058,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
|
|
@ -16063,7 +16067,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
|
|
@ -16071,7 +16076,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
|
|
|
|||
|
|
@ -10600,6 +10600,49 @@
|
|||
"title": "Audio",
|
||||
"type": "array"
|
||||
},
|
||||
"images": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"base64": {
|
||||
"description": "Base64-encoded image bytes",
|
||||
"minLength": 1,
|
||||
"title": "Base64",
|
||||
"type": "string"
|
||||
},
|
||||
"content_type": {
|
||||
"description": "An image/* media type, such as image/png",
|
||||
"pattern": "^image/\\S+$",
|
||||
"title": "Content Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"content_type",
|
||||
"base64"
|
||||
],
|
||||
"title": "DecisionsBase64Image",
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Images as data URLs or content_type plus base64, for providers that look at images",
|
||||
"title": "Images"
|
||||
},
|
||||
"model": {
|
||||
"description": "A decision model from the proxy model_list",
|
||||
"title": "Model",
|
||||
|
|
@ -11448,6 +11491,49 @@
|
|||
"title": "Audio",
|
||||
"type": "array"
|
||||
},
|
||||
"images": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"base64": {
|
||||
"description": "Base64-encoded image bytes",
|
||||
"minLength": 1,
|
||||
"title": "Base64",
|
||||
"type": "string"
|
||||
},
|
||||
"content_type": {
|
||||
"description": "An image/* media type, such as image/png",
|
||||
"pattern": "^image/\\S+$",
|
||||
"title": "Content Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"content_type",
|
||||
"base64"
|
||||
],
|
||||
"title": "DecisionsBase64Image",
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Images as data URLs or content_type plus base64, for providers that look at images",
|
||||
"title": "Images"
|
||||
},
|
||||
"model": {
|
||||
"description": "A decision model from the proxy model_list",
|
||||
"title": "Model",
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ async def _invalid_request(
|
|||
)
|
||||
|
||||
|
||||
_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "audio", "videos"})
|
||||
_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "audio", "videos"})
|
||||
|
||||
|
||||
def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
from litellm.decisions.call import (
|
||||
DecisionsAudioClips,
|
||||
DecisionsCall,
|
||||
DecisionsImages,
|
||||
DecisionsVideoClips,
|
||||
SystemOneQuestions,
|
||||
SystemOneRequestFields,
|
||||
|
|
@ -23,6 +24,7 @@ def _prepare_systemone_call(
|
|||
model: str,
|
||||
state: DecisionsJSON | None,
|
||||
questions: SystemOneQuestions | None,
|
||||
images: DecisionsImages | None,
|
||||
safety_identifier: str | None,
|
||||
audio: DecisionsAudioClips | None,
|
||||
videos: DecisionsVideoClips | None,
|
||||
|
|
@ -36,7 +38,7 @@ def _prepare_systemone_call(
|
|||
reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions")
|
||||
return prepare_call(
|
||||
model=model,
|
||||
fields=SystemOneRequestFields(state=state, questions=questions, audio=audio, videos=videos),
|
||||
fields=SystemOneRequestFields(state=state, questions=questions, images=images, audio=audio, videos=videos),
|
||||
safety_identifier=safety_identifier,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
|
|
@ -65,12 +67,14 @@ async def asystemone(
|
|||
safety_identifier: str | None = None,
|
||||
audio: DecisionsAudioClips | None = None,
|
||||
videos: DecisionsVideoClips | None = None,
|
||||
images: DecisionsImages | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> DecisionsResponse:
|
||||
call: Final = _prepare_systemone_call(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
images=images,
|
||||
safety_identifier=safety_identifier,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
|
|
@ -97,12 +101,14 @@ def systemone(
|
|||
safety_identifier: str | None = None,
|
||||
audio: DecisionsAudioClips | None = None,
|
||||
videos: DecisionsVideoClips | None = None,
|
||||
images: DecisionsImages | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> DecisionsResponse:
|
||||
call: Final = _prepare_systemone_call(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
images=images,
|
||||
safety_identifier=safety_identifier,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
|
|
|
|||
|
|
@ -67,6 +67,19 @@ DecisionQuestionMap: TypeAlias = Annotated[
|
|||
]
|
||||
|
||||
|
||||
class DecisionsBase64Image(LiteLLMPydanticObjectBase):
|
||||
content_type: Annotated[str, Field(pattern=r"^image/\S+$", description="An image/* media type, such as image/png")]
|
||||
base64: Annotated[str, Field(min_length=1, description="Base64-encoded image bytes")]
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
DecisionsImage: TypeAlias = (
|
||||
Annotated[str, Field(pattern=r"^(?i:data:image/[^;,\s]+(?:;[^;,\s]+)*;base64,.+|https?://\S+)$")]
|
||||
| DecisionsBase64Image
|
||||
)
|
||||
|
||||
|
||||
class DecisionsBase64Audio(LiteLLMPydanticObjectBase):
|
||||
content_type: Annotated[str, Field(pattern=r"^audio/\S+$", description="An audio/* media type, such as audio/wav")]
|
||||
base64: Annotated[str, Field(min_length=1, description="Base64-encoded audio bytes")]
|
||||
|
|
@ -118,6 +131,10 @@ class DecisionsRequestBody(DecisionsMediaBody):
|
|||
DecisionQuestionMap,
|
||||
Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"),
|
||||
]
|
||||
images: Annotated[
|
||||
Sequence[DecisionsImage] | None,
|
||||
Field(description="Images as data URLs or content_type plus base64, for providers that look at images"),
|
||||
] = None
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
|
@ -131,6 +148,7 @@ class DecisionsCallParams(TypedDict, total=False):
|
|||
model: Required[ReadOnly[str]]
|
||||
state: Required[ReadOnly[DecisionsJSON]]
|
||||
questions: Required[ReadOnly[DecisionQuestionMap]]
|
||||
images: ReadOnly[Sequence[DecisionsImage] | None]
|
||||
audio: ReadOnly[Sequence[DecisionsAudio]]
|
||||
videos: ReadOnly[Sequence[DecisionsVideo]]
|
||||
api_key: ReadOnly[str | None]
|
||||
|
|
@ -419,6 +437,7 @@ _NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({})
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class DecisionsIRState:
|
||||
state: DecisionsJSON
|
||||
images: tuple[DecisionsImage, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -16029,7 +16029,8 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
|
||||
"supports_audio_input": true,
|
||||
"supports_video_input": true
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-omni": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
|
|
@ -16039,7 +16040,8 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
|
||||
"supports_audio_input": true,
|
||||
"supports_video_input": true
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
|
|
@ -16047,7 +16049,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
|
|
@ -16055,7 +16058,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
|
|
@ -16063,7 +16067,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
|
|
@ -16071,7 +16076,8 @@
|
|||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
|
|
|
|||
312
tests/integration/providers/test_cloudflare_clef_images_chaos.py
Normal file
312
tests/integration/providers/test_cloudflare_clef_images_chaos.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from queue import SimpleQueue
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import unquote, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.bedrock_runtime_peer import PNG_1X1
|
||||
from integration._support.client import Gateway, eventually, list_value, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
Route = Literal["/v1/systemone", "/systemone", "/v1/decisions", "/decisions"]
|
||||
|
||||
_ROUTES: Final[tuple[Route, ...]] = ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions")
|
||||
_API_KEY: Final = "synthetic-cloudflare-key"
|
||||
_INPUT_TOKENS: Final = 412
|
||||
_PNG_DATA_URL: Final = f"data:image/png;base64,{base64.b64encode(PNG_1X1).decode()}"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
|
||||
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
|
||||
_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = {"red": {"type": "noul", "instructions": "Is the screenshot red?"}}
|
||||
_OPENAI_QUESTIONS: Final[list[JsonValue]] = [
|
||||
{"type": "predicate", "name": "red", "instructions": "Is the screenshot red?"}
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Call:
|
||||
route: Route
|
||||
marker: str
|
||||
images: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Served:
|
||||
call: _Call
|
||||
status: int
|
||||
text: str
|
||||
|
||||
|
||||
def _state(marker: str) -> str:
|
||||
return f"Is the checkout screenshot red? marker-{marker}"
|
||||
|
||||
|
||||
def _probability(marker: str) -> float:
|
||||
return round(int(marker[:6], 16) / 0x1000000, 7)
|
||||
|
||||
|
||||
def _marker_of(request: Request) -> str:
|
||||
found: Final = _MARKER.search(request.body.decode() or request.target)
|
||||
assert found is not None, (request.target, request.body[:200])
|
||||
return found.group(1)
|
||||
|
||||
|
||||
def _clef_reply(request: Request) -> Reply:
|
||||
reply: Final[dict[str, JsonValue]] = {
|
||||
"result": {
|
||||
"model": "clef",
|
||||
"answers": {"red": {"type": "noul", "noul": _probability(_marker_of(request))}},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
},
|
||||
"success": True,
|
||||
}
|
||||
return Reply(body=json.dumps(reply).encode())
|
||||
|
||||
|
||||
def _is_image_fetch(request: Request) -> bool:
|
||||
return request.method == "GET" and urlsplit(request.target).path == "/image.png"
|
||||
|
||||
|
||||
def _body(call: _Call, model: str) -> dict[str, JsonValue]:
|
||||
if call.route.endswith("systemone"):
|
||||
return {
|
||||
"model": model,
|
||||
"state": _state(call.marker),
|
||||
"questions": _SYSTEM_ONE_QUESTIONS,
|
||||
"images": list(call.images),
|
||||
"num_retries": 0,
|
||||
}
|
||||
parts: Final[list[JsonValue]] = [
|
||||
{"type": "input_text", "text": _state(call.marker)},
|
||||
*({"type": "input_image", "image_url": image} for image in call.images),
|
||||
]
|
||||
return {
|
||||
"model": model,
|
||||
"input": [{"role": "user", "content": parts}],
|
||||
"questions": _OPENAI_QUESTIONS,
|
||||
"num_retries": 0,
|
||||
}
|
||||
|
||||
|
||||
def _upstream_by_marker(received: Sequence[Request]) -> dict[str, dict[str, JsonValue]]:
|
||||
posts: Final = tuple(request for request in received if request.method == "POST")
|
||||
for request in posts:
|
||||
assert unquote(request.target) == "/ai/run/@cf/cloudflare/clef", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers
|
||||
by_marker: Final = {_marker_of(request): _JSON_OBJECT.validate_json(request.body) for request in posts}
|
||||
assert len(by_marker) == len(posts), "a request reached clef twice"
|
||||
return by_marker
|
||||
|
||||
|
||||
def _answered_probability(route: Route, answers: JsonValue) -> JsonValue:
|
||||
if route.endswith("systemone"):
|
||||
return object_value(object_value(answers)["red"])["noul"]
|
||||
(answer,) = list_value(answers)
|
||||
return object_value(answer)["probability"]
|
||||
|
||||
|
||||
def _assert_answered(served: _Served) -> None:
|
||||
assert served.status == 200, served.text
|
||||
answers: Final = _JSON_OBJECT.validate_json(served.text)["answers"]
|
||||
assert _answered_probability(served.call.route, answers) == _probability(served.call.marker), served.text
|
||||
|
||||
|
||||
def _assert_clef_got_inlined_images(call: _Call, body: dict[str, JsonValue]) -> None:
|
||||
assert body == {
|
||||
"model": "clef",
|
||||
"state": _state(call.marker),
|
||||
"questions": _SYSTEM_ONE_QUESTIONS,
|
||||
"images": [_PNG_DATA_URL] * len(call.images),
|
||||
}, (call, body)
|
||||
|
||||
|
||||
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
|
||||
response: Final = await client.post(call.route, json=_body(call, model), headers={"Authorization": f"Bearer {key}"})
|
||||
return _Served(call=call, status=response.status_code, text=response.text)
|
||||
|
||||
|
||||
async def _burst(
|
||||
gateway: Gateway, model: str, calls: Sequence[_Call], *, tolerate_transport_errors: bool = False
|
||||
) -> tuple[_Served, ...]:
|
||||
async with httpx.AsyncClient(base_url=str(gateway.client.base_url), timeout=90, trust_env=False) as client:
|
||||
results: Final = await asyncio.gather(
|
||||
*(_send(client, gateway.key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
|
||||
)
|
||||
for result in results:
|
||||
assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
|
||||
return tuple(result for result in results if isinstance(result, _Served))
|
||||
|
||||
|
||||
def _pooled_database_url(url: str) -> str:
|
||||
parts: Final = urlsplit(url)
|
||||
query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part)
|
||||
return urlunsplit(parts._replace(query=query))
|
||||
|
||||
|
||||
def _config(wire: Wire, tmp_path: Path) -> tuple[Path, str]:
|
||||
model: Final = f"clef-images-chaos-{uuid.uuid4().hex[:8]}"
|
||||
config: Final = _JSON_OBJECT.validate_python(
|
||||
yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
)
|
||||
path: Final = tmp_path / "clef-images-chaos.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**config,
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {"model": "cloudflare/clef", "api_base": wire.url, "api_key": _API_KEY},
|
||||
}
|
||||
],
|
||||
"general_settings": {
|
||||
**object_value(config["general_settings"]),
|
||||
"user_url_allowed_hosts": ["127.0.0.1"],
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return path, model
|
||||
|
||||
|
||||
def _successful_spend_rows(model: str, expected: int) -> int:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
|
||||
lambda found: sum(row["status"] == "success" for row in found) >= expected,
|
||||
seconds=70,
|
||||
)
|
||||
assert len({row["request_id"] for row in rows}) == len(rows), rows
|
||||
return sum(row["status"] == "success" for row in rows)
|
||||
|
||||
|
||||
def _open_upstream_connections(pid: int, upstream: str) -> int:
|
||||
port: Final = urlsplit(upstream).port
|
||||
return sum(
|
||||
1
|
||||
for connection in psutil.Process(pid).net_connections(kind="tcp")
|
||||
if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
|
||||
)
|
||||
|
||||
|
||||
def _mixed_calls(wire: Wire, count: int) -> tuple[_Call, ...]:
|
||||
def images(index: int, marker: str) -> tuple[str, ...]:
|
||||
remote: Final = f"{wire.url}/image.png?id=marker-{marker}"
|
||||
return ((_PNG_DATA_URL,), (remote,), (remote, _PNG_DATA_URL, remote, _PNG_DATA_URL))[index % 3]
|
||||
|
||||
markers: Final = tuple(uuid.uuid4().hex for _ in range(count))
|
||||
return tuple(
|
||||
_Call(route=_ROUTES[index % len(_ROUTES)], marker=marker, images=images(index, marker))
|
||||
for index, marker in enumerate(markers)
|
||||
)
|
||||
|
||||
|
||||
def _overflow_calls(wire: Wire, count: int) -> tuple[_Call, ...]:
|
||||
markers: Final = tuple(uuid.uuid4().hex for _ in range(count))
|
||||
return tuple(
|
||||
_Call(
|
||||
route=_ROUTES[index % len(_ROUTES)], marker=marker, images=(f"{wire.url}/image.png?id=marker-{marker}",) * 5
|
||||
)
|
||||
for index, marker in enumerate(markers)
|
||||
)
|
||||
|
||||
|
||||
def _image_host_and_clef(request: Request) -> Reply:
|
||||
if _is_image_fetch(request):
|
||||
return Reply(body=PNG_1X1, content_type="image/png")
|
||||
return _clef_reply(request)
|
||||
|
||||
|
||||
@pytest.mark.timeout(600)
|
||||
async def test_a_mixed_image_burst_across_two_workers_reaches_clef_once_per_request(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with wire_server(_image_host_and_clef) as wire:
|
||||
path, model = _config(wire, tmp_path)
|
||||
calls: Final = _mixed_calls(wire, 24)
|
||||
overflow: Final = _overflow_calls(wire, 8)
|
||||
overrides: Final = {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}
|
||||
with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned:
|
||||
served: Final = await _burst(owned.gateway, model, (*calls, *overflow))
|
||||
assert len(served) == 32
|
||||
overflow_markers: Final = frozenset(call.marker for call in overflow)
|
||||
for item in served:
|
||||
if item.call.marker in overflow_markers:
|
||||
assert item.status == 400, item.text
|
||||
assert "at most 4 images per request, got 5" in item.text, item.text
|
||||
else:
|
||||
_assert_answered(item)
|
||||
received: Final = wire.drain()
|
||||
fetched: Final = sorted(_marker_of(request) for request in received if _is_image_fetch(request))
|
||||
assert overflow_markers.isdisjoint(fetched), fetched
|
||||
by_marker: Final = _upstream_by_marker(received)
|
||||
assert sorted(by_marker) == sorted(call.marker for call in calls)
|
||||
for call in calls:
|
||||
_assert_clef_got_inlined_images(call, by_marker[call.marker])
|
||||
assert _successful_spend_rows(model, 24) == 24
|
||||
|
||||
|
||||
@pytest.mark.timeout(600)
|
||||
async def test_worker_sigkill_while_images_are_fetched_leaves_the_sibling_serving(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
release: Final = threading.Event()
|
||||
held_fetches: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if not _is_image_fetch(request):
|
||||
return _clef_reply(request)
|
||||
held_fetches.put(_marker_of(request))
|
||||
assert release.wait(timeout=60), "The image fetches were never released"
|
||||
return Reply(body=PNG_1X1, content_type="image/png")
|
||||
|
||||
with wire_server(respond) as wire:
|
||||
path, model = _config(wire, tmp_path)
|
||||
calls: Final = tuple(
|
||||
_Call(route="/v1/systemone", marker=marker, images=(f"{wire.url}/image.png?id=marker-{marker}",))
|
||||
for marker in (uuid.uuid4().hex for _ in range(16))
|
||||
)
|
||||
overrides: Final = {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}
|
||||
with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned:
|
||||
workers: Final = eventually(
|
||||
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
|
||||
lambda pids: len(pids) == 2,
|
||||
seconds=30,
|
||||
)
|
||||
burst: Final = asyncio.create_task(_burst(owned.gateway, model, calls, tolerate_transport_errors=True))
|
||||
await asyncio.to_thread(eventually, held_fetches.qsize, lambda size: size == 16, 60)
|
||||
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
|
||||
assert sum(held_by.values()) == 16, held_by
|
||||
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
|
||||
with owned.replacing_worker(victim_pid) as kill:
|
||||
kill.freeze()
|
||||
await asyncio.to_thread(kill)
|
||||
release.set()
|
||||
served: Final = await burst
|
||||
assert held_by[survivor_pid] >= 8, held_by
|
||||
assert len(served) == held_by[survivor_pid], (held_by, len(served))
|
||||
for item in served:
|
||||
_assert_answered(item)
|
||||
follow_up: Final = _Call(route="/v1/decisions", marker=uuid.uuid4().hex, images=(_PNG_DATA_URL,))
|
||||
(answered,) = await _burst(owned.gateway, model, (follow_up,))
|
||||
_assert_answered(answered)
|
||||
by_marker: Final = _upstream_by_marker(wire.drain())
|
||||
assert sorted(by_marker) == sorted((*(item.call.marker for item in served), follow_up.marker))
|
||||
for item in served:
|
||||
_assert_clef_got_inlined_images(item.call, by_marker[item.call.marker])
|
||||
_assert_clef_got_inlined_images(follow_up, by_marker[follow_up.marker])
|
||||
877
tests/integration/providers/test_cloudflare_clef_images_wire.py
Normal file
877
tests/integration/providers/test_cloudflare_clef_images_wire.py
Normal file
|
|
@ -0,0 +1,877 @@
|
|||
import base64
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import unquote, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.bedrock_runtime_peer import PNG_1X1
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment, list_value, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.timing import assert_within_factor_of_control, timed
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
Route = Literal["/v1/systemone", "/systemone", "/v1/decisions", "/decisions"]
|
||||
|
||||
_ROUTES: Final[tuple[Route, ...]] = ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions")
|
||||
_API_KEY: Final = "synthetic-cloudflare-key"
|
||||
_CLEF: Final = "cloudflare/clef"
|
||||
_ALLOWLISTED_MODEL: Final = "clef-images-allowlisted"
|
||||
_INPUT_TOKENS: Final = 412
|
||||
_PNG_BASE64: Final = base64.b64encode(PNG_1X1).decode()
|
||||
_PNG_DATA_URL: Final = f"data:image/png;base64,{_PNG_BASE64}"
|
||||
_JPEG_DATA_URL: Final = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQ=="
|
||||
_PNG_OBJECT: Final[dict[str, JsonValue]] = {"content_type": "image/png", "base64": _PNG_BASE64}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_JSON_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
|
||||
_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = {"red": {"type": "noul", "instructions": "Is the screenshot red?"}}
|
||||
_OPENAI_QUESTIONS: Final[list[JsonValue]] = [
|
||||
{"type": "predicate", "name": "red", "instructions": "Is the screenshot red?"}
|
||||
]
|
||||
_SPEND_QUERY: Final = (
|
||||
'SELECT spend, status, call_type, model_group, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" '
|
||||
"WHERE request_id = %s"
|
||||
)
|
||||
_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120)
|
||||
_COLD_OPENAPI_SECONDS: Final = 120
|
||||
_SLOW_IMAGE_RELEASE: Final = threading.Event()
|
||||
_IMAGE_INPUT_RECORDER: Final = """
|
||||
import json
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
|
||||
class ImageInputRecorder(CustomLogger):
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
payload = {"call_id": kwargs.get("litellm_call_id"), "input": kwargs.get("input")}
|
||||
async with httpx.AsyncClient(trust_env=False, timeout=10) as client:
|
||||
await client.post(os.environ["IMAGE_INPUT_SINK"], content=json.dumps(payload, default=str))
|
||||
|
||||
|
||||
instance = ImageInputRecorder()
|
||||
"""
|
||||
|
||||
|
||||
def _marker_of(request: Request) -> str:
|
||||
found: Final = _MARKER.search(request.body.decode())
|
||||
assert found is not None, request.body[:200]
|
||||
return found.group(1)
|
||||
|
||||
|
||||
def _state(marker: str) -> str:
|
||||
return f"Is the checkout screenshot red? marker-{marker}"
|
||||
|
||||
|
||||
def _probability(marker: str) -> float:
|
||||
return round(int(marker[:6], 16) / 0x1000000, 7)
|
||||
|
||||
|
||||
def _clef_name(request: Request) -> str:
|
||||
return unquote(request.target).rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def _clef_reply(request: Request) -> Reply:
|
||||
reply: Final[dict[str, JsonValue]] = {
|
||||
"result": {
|
||||
"model": _clef_name(request),
|
||||
"answers": {"red": {"type": "noul", "noul": _probability(_marker_of(request))}},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
},
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"messages": [],
|
||||
}
|
||||
return Reply(body=json.dumps(reply).encode())
|
||||
|
||||
|
||||
def _speaks_system_one(route: Route) -> bool:
|
||||
return route.endswith("systemone")
|
||||
|
||||
|
||||
def _openai_input(text: str, images: Sequence[JsonValue]) -> list[JsonValue]:
|
||||
text_parts: Final[tuple[JsonValue, ...]] = ({"type": "input_text", "text": text},) if text else ()
|
||||
image_parts: Final = tuple({"type": "input_image", "image_url": image} for image in images)
|
||||
return [{"role": "user", "content": [*text_parts, *image_parts]}]
|
||||
|
||||
|
||||
def _body(
|
||||
route: Route, model: str, marker: str, images: Sequence[JsonValue], **extra: JsonValue
|
||||
) -> dict[str, JsonValue]:
|
||||
if _speaks_system_one(route):
|
||||
return {
|
||||
"model": model,
|
||||
"state": _state(marker),
|
||||
"questions": _SYSTEM_ONE_QUESTIONS,
|
||||
"images": list(images),
|
||||
**extra,
|
||||
}
|
||||
return {"model": model, "input": _openai_input(_state(marker), images), "questions": _OPENAI_QUESTIONS, **extra}
|
||||
|
||||
|
||||
def _text_only_body(route: Route, model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]:
|
||||
if _speaks_system_one(route):
|
||||
return {"model": model, "state": _state(marker), "questions": _SYSTEM_ONE_QUESTIONS, **extra}
|
||||
return {"model": model, "input": _state(marker), "questions": _OPENAI_QUESTIONS, **extra}
|
||||
|
||||
|
||||
def _expected_response(route: Route, clef_name: str, marker: str) -> dict[str, JsonValue]:
|
||||
if _speaks_system_one(route):
|
||||
return {
|
||||
"model": clef_name,
|
||||
"answers": {"red": {"type": "noul", "noul": _probability(marker)}},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
}
|
||||
return {
|
||||
"model": clef_name,
|
||||
"answers": [{"type": "predicate", "name": "red", "probability": _probability(marker)}],
|
||||
"usage": {
|
||||
"input_tokens": _INPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": _INPUT_TOKENS,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _expected_upstream_body(clef_name: str, state: str, images: Sequence[JsonValue] = ()) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": clef_name,
|
||||
"state": state,
|
||||
"questions": _SYSTEM_ONE_QUESTIONS,
|
||||
**({"images": list(images)} if images else {}),
|
||||
}
|
||||
|
||||
|
||||
def _upstream_body(request: Request, clef_name: str = "clef") -> dict[str, JsonValue]:
|
||||
assert request.method == "POST", (request.method, request.target)
|
||||
assert unquote(request.target) == f"/ai/run/@cf/cloudflare/{clef_name}", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers
|
||||
return _JSON_OBJECT.validate_json(request.body)
|
||||
|
||||
|
||||
def _decide(
|
||||
gateway: Gateway, route: Route, model: str, marker: str, images: Sequence[JsonValue], **extra: JsonValue
|
||||
) -> httpx.Response:
|
||||
return gateway.request("POST", route, _body(route, model, marker, images, **extra))
|
||||
|
||||
|
||||
def _number(value: JsonValue) -> float:
|
||||
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
|
||||
return float(value)
|
||||
|
||||
|
||||
def _expected_spend(cost_map_key: str) -> float:
|
||||
cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes())
|
||||
priced: Final = cost_map.get(cost_map_key)
|
||||
assert priced is not None, f"{cost_map_key} has no cost map entry"
|
||||
return _INPUT_TOKENS * _number(object_value(priced)["input_cost_per_token"])
|
||||
|
||||
|
||||
def _spend_row(call_id: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _call_type(route: Route) -> str:
|
||||
return "asystemone" if _speaks_system_one(route) else "adecisions"
|
||||
|
||||
|
||||
def _assert_billed(gateway: Gateway, response: httpx.Response, model: str, route: Route, cost_map_key: str) -> None:
|
||||
expected: Final = _expected_spend(cost_map_key)
|
||||
assert math.isclose(float(response.headers["x-litellm-response-cost"]), expected, rel_tol=1e-9), response.headers
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
row: Final = _spend_row(call_id)
|
||||
assert (row["status"], row["call_type"], row["model_group"], row["prompt_tokens"], row["completion_tokens"]) == (
|
||||
"success",
|
||||
_call_type(route),
|
||||
model,
|
||||
_INPUT_TOKENS,
|
||||
0,
|
||||
), row
|
||||
assert math.isclose(_number(row["spend"]), expected, rel_tol=1e-9), row
|
||||
listed: Final = gateway.request("GET", "/spend/logs", params={"request_id": call_id})
|
||||
assert listed.status_code == 200, listed.text
|
||||
(reported,) = _JSON_ROWS.validate_json(listed.content)
|
||||
assert (reported["request_id"], reported["model_group"], reported["prompt_tokens"]) == (
|
||||
call_id,
|
||||
model,
|
||||
_INPUT_TOKENS,
|
||||
), reported
|
||||
assert math.isclose(_number(reported["spend"]), expected, rel_tol=1e-9), reported
|
||||
|
||||
|
||||
def _images_for(route: Route) -> list[JsonValue]:
|
||||
return [_PNG_DATA_URL, _PNG_OBJECT] if _speaks_system_one(route) else [_PNG_DATA_URL, _JPEG_DATA_URL]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", _ROUTES)
|
||||
def test_images_reach_clef_in_its_images_field_on_every_route_and_are_billed(gateway: Gateway, route: Route) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
images: Final = _images_for(route)
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _decide(gateway, route, model, marker, images)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response(route, "clef", marker)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), images)
|
||||
_assert_billed(gateway, response, model, route, "cloudflare/@cf/cloudflare/clef")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("spelling", "clef_name"),
|
||||
(("cloudflare/clef-flash", "clef-flash"), ("cloudflare/@cf/cloudflare/clef-omni", "clef-omni")),
|
||||
ids=("clef-flash", "clef-omni"),
|
||||
)
|
||||
def test_clef_flash_and_clef_omni_take_images_and_carry_their_own_price(
|
||||
gateway: Gateway, spelling: str, clef_name: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=spelling, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL])
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response("/v1/systemone", clef_name, marker)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request, clef_name) == _expected_upstream_body(clef_name, _state(marker), [_PNG_DATA_URL])
|
||||
_assert_billed(gateway, response, model, "/v1/systemone", spelling)
|
||||
|
||||
|
||||
def test_an_image_only_message_reaches_clef_with_an_empty_state(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
reply: Final = json.dumps(
|
||||
{
|
||||
"result": {
|
||||
"model": "clef",
|
||||
"answers": {"red": {"type": "noul", "noul": _probability(marker)}},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
},
|
||||
"success": True,
|
||||
}
|
||||
).encode()
|
||||
with wire_server(lambda _: Reply(body=reply)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/decisions",
|
||||
{"model": model, "input": _openai_input("", [_PNG_DATA_URL]), "questions": _OPENAI_QUESTIONS},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response("/v1/decisions", "clef", marker)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", "", [_PNG_DATA_URL])
|
||||
|
||||
|
||||
def _luna_reply(request: Request) -> Reply:
|
||||
reply: Final[dict[str, JsonValue]] = {
|
||||
"model": "gpt-6-luna",
|
||||
"answers": [{"type": "predicate", "name": "red", "probability": _probability(_marker_of(request))}],
|
||||
"usage": {
|
||||
"input_tokens": _INPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": _INPUT_TOKENS,
|
||||
},
|
||||
}
|
||||
return Reply(body=json.dumps(reply).encode())
|
||||
|
||||
|
||||
def test_an_openai_decision_model_gets_system_one_images_as_input_image_parts(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_luna_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/gpt-6-luna", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL, _PNG_OBJECT])
|
||||
assert response.status_code == 200, response.text
|
||||
assert object_value(object_value(response.json()["answers"])["red"])["noul"] == _probability(marker)
|
||||
(request,) = wire.drain()
|
||||
assert unquote(request.target) == "/v1/decisions", request.target
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"model": "gpt-6-luna",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"type": "message",
|
||||
"content": [
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL},
|
||||
{"type": "input_text", "text": _state(marker)},
|
||||
],
|
||||
}
|
||||
],
|
||||
"questions": _OPENAI_QUESTIONS,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions"))
|
||||
def test_more_images_than_clef_takes_are_refused_before_any_download_or_upstream_call(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
remote: Final = [f"{wire.url}/image.png?n={index}&id=marker-{marker}" for index in range(5)]
|
||||
for model in (strict, dropping):
|
||||
refused = _decide(gateway, route, model, marker, remote)
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert "at most 4 images per request, got 5" in refused.text, refused.text
|
||||
assert wire.drain() == ()
|
||||
four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL]
|
||||
accepted: Final = _decide(gateway, route, strict, marker, four)
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), four)
|
||||
|
||||
|
||||
_MALFORMED: Final[tuple[tuple[str, JsonValue], ...]] = (
|
||||
("bare_string", _PNG_DATA_URL),
|
||||
("number", 5),
|
||||
("object_not_list", _PNG_OBJECT),
|
||||
("url_object", [{"url": _PNG_DATA_URL}]),
|
||||
("chat_image_url", [{"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}]),
|
||||
("extra_key", [{**_PNG_OBJECT, "detail": "high"}]),
|
||||
("missing_base64", [{"content_type": "image/png"}]),
|
||||
("non_image_type", [{"content_type": "text/plain", "base64": "AA=="}]),
|
||||
("empty_base64", [{"content_type": "image/png", "base64": ""}]),
|
||||
("not_a_url", ["a" * 5000]),
|
||||
("non_image_data_url", ["data:text/plain;base64,AA=="]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "images"), _MALFORMED, ids=[label for label, _ in _MALFORMED])
|
||||
def test_malformed_images_are_refused_unless_the_deployment_drops_params(
|
||||
gateway: Gateway, label: str, images: JsonValue
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
body: Final = {**_text_only_body("/v1/systemone", strict, marker), "images": images}
|
||||
refused: Final = gateway.request("POST", "/v1/systemone", body)
|
||||
assert refused.status_code == 400, (label, refused.text)
|
||||
assert "Invalid Decisions request" in refused.text and "images" in refused.text, (label, refused.text)
|
||||
assert wire.drain() == (), label
|
||||
dropped: Final = gateway.request("POST", "/v1/systemone", {**body, "model": dropping})
|
||||
assert dropped.status_code == 200, (label, dropped.text)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker)), label
|
||||
|
||||
|
||||
def test_a_malformed_image_under_drop_params_keeps_the_well_formed_ones(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
images: Final[list[JsonValue]] = [
|
||||
{"content_type": "text/plain", "base64": "AA=="},
|
||||
_PNG_DATA_URL,
|
||||
{"url": _PNG_DATA_URL},
|
||||
_PNG_OBJECT,
|
||||
]
|
||||
response: Final = _decide(gateway, "/v1/systemone", dropping, marker, images)
|
||||
assert response.status_code == 200, response.text
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL, _PNG_OBJECT])
|
||||
|
||||
|
||||
_MALFORMED_IMAGE_URLS: Final = (
|
||||
("not_a_url", "not-an-image"),
|
||||
("pdf_data_url", "data:application/pdf;base64,AA=="),
|
||||
("ftp_url", "ftp://images.example/a.png"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label", "image_url"), _MALFORMED_IMAGE_URLS, ids=[label for label, _ in _MALFORMED_IMAGE_URLS]
|
||||
)
|
||||
def test_a_malformed_input_image_is_refused_unless_the_deployment_drops_params(
|
||||
gateway: Gateway, label: str, image_url: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
refused: Final = _decide(gateway, "/v1/decisions", strict, marker, [image_url, _PNG_DATA_URL])
|
||||
assert refused.status_code == 400, (label, refused.text)
|
||||
assert "Invalid Decisions request: 1 input_image" in refused.text, (label, refused.text)
|
||||
assert wire.drain() == (), label
|
||||
dropped: Final = _decide(gateway, "/v1/decisions", dropping, marker, [image_url, _PNG_DATA_URL])
|
||||
assert dropped.status_code == 200, (label, dropped.text)
|
||||
assert dropped.json() == _expected_response("/v1/decisions", "clef", marker), label
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]), label
|
||||
|
||||
|
||||
_GIF_BASE64: Final = "R0lGODlhAQABAAAAACw="
|
||||
_PNG_DATA_URL_WITH_NAME: Final = f"data:IMAGE/PNG;name=screen.png;base64,{_PNG_BASE64}"
|
||||
_WEBP_DATA_URL: Final = "data:image/webp;base64,UklGRhoAAABXRUJQ"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions"))
|
||||
def test_clef_gets_png_jpeg_and_webp_images_without_media_type_parameters(gateway: Gateway, route: Route) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
images: Final = [_PNG_DATA_URL_WITH_NAME, _JPEG_DATA_URL, _WEBP_DATA_URL]
|
||||
response: Final = _decide(gateway, route, model, marker, images)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response(route, "clef", marker)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body(
|
||||
"clef", _state(marker), [_PNG_DATA_URL, _JPEG_DATA_URL, _WEBP_DATA_URL]
|
||||
)
|
||||
|
||||
|
||||
def test_an_openai_decision_model_gets_a_data_url_with_media_type_parameters_unchanged(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_luna_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/gpt-6-luna", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _decide(gateway, "/v1/decisions", model, marker, [_PNG_DATA_URL_WITH_NAME])
|
||||
assert response.status_code == 200, response.text
|
||||
(request,) = wire.drain()
|
||||
sent: Final = list_value(
|
||||
object_value(list_value(_JSON_OBJECT.validate_json(request.body)["input"])[0])["content"]
|
||||
)
|
||||
assert sent == [
|
||||
{"type": "input_text", "text": _state(marker)},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL_WITH_NAME},
|
||||
]
|
||||
|
||||
|
||||
_REFUSED_TYPES: Final[tuple[tuple[str, Route, JsonValue], ...]] = (
|
||||
("gif_data_url", "/v1/decisions", f"data:image/gif;base64,{_GIF_BASE64}"),
|
||||
("gif_object", "/v1/systemone", {"content_type": "image/gif", "base64": _GIF_BASE64}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "route", "image"), _REFUSED_TYPES, ids=[label for label, _, _ in _REFUSED_TYPES])
|
||||
def test_an_image_type_clef_refuses_is_refused_unless_the_deployment_drops_params(
|
||||
gateway: Gateway, label: str, route: Route, image: JsonValue
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
refused: Final = _decide(gateway, route, strict, marker, [image, _PNG_DATA_URL])
|
||||
assert refused.status_code == 400, (label, refused.text)
|
||||
assert (
|
||||
"cloudflare takes only image/jpeg, image/jpg, image/png, image/webp images, got image/gif" in refused.text
|
||||
), (label, refused.text)
|
||||
assert wire.drain() == (), label
|
||||
dropped: Final = _decide(gateway, route, dropping, marker, [image, _PNG_DATA_URL])
|
||||
assert dropped.status_code == 200, (label, dropped.text)
|
||||
assert dropped.json() == _expected_response(route, "clef", marker), label
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL]), label
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions"))
|
||||
def test_an_image_clef_refuses_does_not_count_toward_its_limit_when_the_deployment_drops_params(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
four: Final = [_PNG_DATA_URL, _JPEG_DATA_URL, _PNG_DATA_URL, _JPEG_DATA_URL]
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
accepted: Final = _decide(gateway, route, dropping, marker, [f"data:image/gif;base64,{_GIF_BASE64}", *four])
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
assert accepted.json() == _expected_response(route, "clef", marker)
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), four)
|
||||
|
||||
|
||||
_MIB: Final = 1024 * 1024
|
||||
|
||||
|
||||
def _png_data_url(decoded_bytes: int) -> str:
|
||||
return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}"
|
||||
|
||||
|
||||
def _png_object(decoded_bytes: int) -> dict[str, JsonValue]:
|
||||
return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()}
|
||||
|
||||
|
||||
_OVERSIZED: Final[tuple[tuple[str, Route, tuple[JsonValue, ...], str], ...]] = (
|
||||
("one_over_4mib", "/v1/systemone", (_png_object(4 * _MIB + 1),), "got one of 4,194,305 bytes"),
|
||||
("one_over_4mib", "/v1/decisions", (_png_data_url(4 * _MIB + 1),), "got one of 4,194,305 bytes"),
|
||||
(
|
||||
"total_over_8mib",
|
||||
"/v1/systemone",
|
||||
(_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)),
|
||||
"got 8,388,609 bytes",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label", "route", "images", "refusal"), _OVERSIZED, ids=[f"{label}{route}" for label, route, _, _ in _OVERSIZED]
|
||||
)
|
||||
def test_images_over_clefs_size_limits_are_refused_before_the_upstream_call(
|
||||
gateway: Gateway, label: str, route: Route, images: tuple[JsonValue, ...], refusal: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
for model in (strict, dropping):
|
||||
refused = _decide(gateway, route, model, marker, images)
|
||||
assert refused.status_code == 400, (label, refused.text[:500])
|
||||
assert refusal in refused.text, (label, refused.text[:500])
|
||||
assert wire.drain() == (), label
|
||||
at_limits: Final[list[JsonValue]] = [_png_data_url(4 * _MIB), _png_object(4 * _MIB)]
|
||||
accepted: Final = _decide(gateway, "/v1/systemone", strict, marker, at_limits)
|
||||
assert accepted.status_code == 200, accepted.text[:500]
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), at_limits), label
|
||||
|
||||
|
||||
def _raw_system_one(gateway: Gateway, model: str, marker: str, images_fragment: str) -> httpx.Response:
|
||||
raw: Final = (
|
||||
f'{{"model": {json.dumps(model)}, "state": {json.dumps(_state(marker))}, '
|
||||
f'"questions": {json.dumps(_SYSTEM_ONE_QUESTIONS)}, {images_fragment}}}'
|
||||
)
|
||||
return gateway.client.post(
|
||||
"/v1/systemone",
|
||||
content=raw.encode(),
|
||||
headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
|
||||
def test_a_duplicated_images_key_is_judged_by_its_last_value(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
malformed: Final = json.dumps([{"content_type": "text/plain", "base64": "AA=="}])
|
||||
good: Final = json.dumps([_PNG_DATA_URL])
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
refused: Final = _raw_system_one(gateway, model, marker, f'"images": {good}, "images": {malformed}')
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert "Invalid Decisions request" in refused.text, refused.text
|
||||
assert wire.drain() == ()
|
||||
accepted: Final = _raw_system_one(gateway, model, marker, f'"images": {malformed}, "images": {good}')
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL])
|
||||
|
||||
|
||||
def test_images_next_to_input_are_refused_unless_params_are_dropped(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
refused: Final = gateway.request(
|
||||
"POST", "/v1/decisions", _text_only_body("/v1/decisions", strict, marker, images=[_PNG_DATA_URL])
|
||||
)
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert "images belongs to the System One format" in refused.text, refused.text
|
||||
assert wire.drain() == ()
|
||||
for accepted in (
|
||||
gateway.request(
|
||||
"POST", "/v1/decisions", _text_only_body("/v1/decisions", dropping, marker, images=[_PNG_DATA_URL])
|
||||
),
|
||||
gateway.request(
|
||||
"POST",
|
||||
"/v1/decisions",
|
||||
_text_only_body("/v1/decisions", strict, marker, images=[_PNG_DATA_URL], drop_params=True),
|
||||
),
|
||||
):
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
assert accepted.json() == _expected_response("/v1/decisions", "clef", marker)
|
||||
assert [_upstream_body(request) for request in wire.drain()] == [
|
||||
_expected_upstream_body("clef", _state(marker))
|
||||
] * 2
|
||||
|
||||
|
||||
def test_a_remote_image_on_the_shared_proxy_is_refused_before_any_fetch(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
for route in ("/v1/systemone", "/v1/decisions"):
|
||||
refused = _decide(gateway, route, model, marker, [f"{wire.url}/image.png?id=marker-{marker}"])
|
||||
assert refused.status_code == 400, (route, refused.text)
|
||||
assert "Unable to fetch image from URL" in refused.text, (route, refused.text)
|
||||
assert "user_url_allowed_hosts" in refused.text, (route, refused.text)
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_a_clef_rejection_of_an_image_reaches_the_caller_like_a_text_only_one(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
message: Final = "Image could not be decoded"
|
||||
rejection: Final = json.dumps(
|
||||
{"result": {}, "success": False, "errors": [{"code": 5006, "message": message}], "messages": []}
|
||||
).encode()
|
||||
garbage: Final = "data:image/png;base64," + "a" * 5000
|
||||
with wire_server(lambda _: Reply(status=422, body=rejection)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
text_only: Final = gateway.request(
|
||||
"POST", "/v1/systemone", _text_only_body("/v1/systemone", model, marker, num_retries=0)
|
||||
)
|
||||
with_image: Final = _decide(gateway, "/v1/systemone", model, marker, [garbage], num_retries=0)
|
||||
for response in (text_only, with_image):
|
||||
assert response.status_code >= 400, response.text
|
||||
assert message in response.text and "answers" not in response.text, response.text
|
||||
assert with_image.status_code == text_only.status_code, (text_only.status_code, with_image.status_code)
|
||||
first, second = (_upstream_body(request) for request in wire.drain())
|
||||
assert first == _expected_upstream_body("clef", _state(marker))
|
||||
assert second == _expected_upstream_body("clef", _state(marker), [garbage])
|
||||
|
||||
|
||||
def test_an_unauthenticated_image_request_is_refused_on_every_route(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
for route in _ROUTES:
|
||||
response = gateway.client.post(route, json=_body(route, model, marker, [_PNG_DATA_URL]))
|
||||
assert response.status_code == 401, (route, response.text)
|
||||
assert "Authentication Error" in response.text, response.text
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_unrelated_chat_keeps_working_after_a_run_of_refused_image_requests(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
control: Final = scenario.model()
|
||||
for _, images in _MALFORMED:
|
||||
refused = gateway.request(
|
||||
"POST", "/v1/systemone", {**_text_only_body("/v1/systemone", model, marker), "images": images}
|
||||
)
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL] * 5).status_code == 400
|
||||
assert wire.drain() == ()
|
||||
assert gateway.chat(control)["choices"], "the control deployment stopped answering"
|
||||
accepted: Final = _decide(gateway, "/v1/systemone", model, marker, [_PNG_DATA_URL])
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == _expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL])
|
||||
|
||||
|
||||
def _timed_liveliness(gateway: Gateway) -> tuple[int, float]:
|
||||
started: Final = time.monotonic()
|
||||
response: Final = gateway.request("GET", "/health/liveliness")
|
||||
return response.status_code, time.monotonic() - started
|
||||
|
||||
|
||||
_WORST_CASE_IMAGES: Final = (
|
||||
("unterminated_data_url", "DATA:IMAGE/" + "x" * 2_000_000),
|
||||
("url_with_trailing_space", "HTTPS://" + "x" * 2_000_000 + " "),
|
||||
("parameters_without_base64", "DATA:IMAGE/PNG" + ";x" * 1_000_000),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "image"), _WORST_CASE_IMAGES, ids=[label for label, _ in _WORST_CASE_IMAGES])
|
||||
def test_the_worst_case_image_string_is_refused_in_time_while_health_keeps_answering(
|
||||
gateway: Gateway, label: str, image: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clef_reply) as wire, gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=1) as pool:
|
||||
model: Final = scenario.model(model=_CLEF, api_base=wire.url, api_key=_API_KEY)
|
||||
control: Final = timed(lambda: _decide(gateway, "/v1/systemone", model, marker, ["x" * len(image)]))
|
||||
assert control.value.status_code == 400, control.value.text
|
||||
liveliness: Final = pool.submit(_timed_liveliness, gateway)
|
||||
refused: Final = timed(lambda: _decide(gateway, "/v1/systemone", model, marker, [image]))
|
||||
health_status, health_elapsed = liveliness.result()
|
||||
assert refused.value.status_code == 400, (label, refused.value.text[:500])
|
||||
assert "Invalid Decisions request" in refused.value.text, (label, refused.value.text[:500])
|
||||
assert_within_factor_of_control(refused.seconds, control=control.seconds, factor=5)
|
||||
assert health_status == 200, (health_status, health_elapsed)
|
||||
assert_within_factor_of_control(health_elapsed, control=control.seconds, factor=5)
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def _image_object_schema(schema: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
images: Final = object_value(object_value(schema["properties"])["images"])
|
||||
array: Final = next(
|
||||
object_value(branch) for branch in list_value(images["anyOf"]) if object_value(branch).get("type") == "array"
|
||||
)
|
||||
entries: Final = list_value(object_value(array["items"])["anyOf"])
|
||||
return next(object_value(entry) for entry in entries if object_value(entry).get("type") == "object")
|
||||
|
||||
|
||||
def test_openapi_documents_what_an_image_object_must_carry(gateway: Gateway) -> None:
|
||||
response: Final = gateway.client.get(
|
||||
"/openapi.json", headers={"Authorization": f"Bearer {gateway.key}"}, timeout=_COLD_OPENAPI_SECONDS
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
paths: Final = object_value(_JSON_OBJECT.validate_json(response.content)["paths"])
|
||||
for route in ("/v1/systemone", "/systemone"):
|
||||
post: Final = object_value(object_value(paths[route])["post"])
|
||||
content: Final = object_value(object_value(post["requestBody"])["content"])
|
||||
schema: Final = object_value(object_value(content["application/json"])["schema"])
|
||||
properties: Final = object_value(_image_object_schema(schema)["properties"])
|
||||
assert object_value(properties["content_type"])["pattern"] == r"^image/\S+$", (route, properties)
|
||||
assert object_value(properties["base64"])["minLength"] == 1, (route, properties)
|
||||
|
||||
|
||||
def _pooled_database_url(url: str) -> str:
|
||||
parts: Final = urlsplit(url)
|
||||
query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part)
|
||||
return urlunsplit(parts._replace(query=query))
|
||||
|
||||
|
||||
def _image_host_and_clef(request: Request) -> Reply:
|
||||
path: Final = urlsplit(request.target).path
|
||||
if request.method == "GET" and path == "/image.png":
|
||||
return Reply(body=PNG_1X1, content_type="image/png")
|
||||
if request.method == "GET" and path == "/page.html":
|
||||
return Reply(body=b"<html></html>", content_type="text/html; charset=utf-8")
|
||||
if request.method == "GET" and path == "/slow.png":
|
||||
_SLOW_IMAGE_RELEASE.wait(timeout=30)
|
||||
return Reply(body=PNG_1X1, content_type="image/png")
|
||||
if request.method == "GET":
|
||||
return Reply(status=404, body=b"not found", content_type="text/plain")
|
||||
return _clef_reply(request)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig() -> Iterator[Gateway]:
|
||||
with gateway_from_environment() as gateway:
|
||||
yield gateway
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def wire() -> Iterator[Wire]:
|
||||
with wire_server(_image_host_and_clef) as served:
|
||||
yield served
|
||||
_SLOW_IMAGE_RELEASE.set()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def sink() -> Iterator[Wire]:
|
||||
with wire_server(lambda _: Reply()) as served:
|
||||
yield served
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def allowlisted(rig: Gateway, wire: Wire, sink: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("clef-images-allowlisted")
|
||||
(directory / "image_input_recorder.py").write_text(_IMAGE_INPUT_RECORDER)
|
||||
config: Final = _JSON_OBJECT.validate_python(
|
||||
yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
)
|
||||
path: Final = directory / "clef-images-allowlisted.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**config,
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _ALLOWLISTED_MODEL,
|
||||
"litellm_params": {"model": _CLEF, "api_base": wire.url, "api_key": _API_KEY},
|
||||
}
|
||||
],
|
||||
"litellm_settings": {
|
||||
**object_value(config["litellm_settings"]),
|
||||
"callbacks": ["image_input_recorder.instance"],
|
||||
},
|
||||
"general_settings": {
|
||||
**object_value(config["general_settings"]),
|
||||
"user_url_allowed_hosts": ["127.0.0.1"],
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
overrides: Final = {
|
||||
"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"]),
|
||||
"IMAGE_INPUT_SINK": sink.url,
|
||||
}
|
||||
with owned_proxy(rig, directory, overrides, config=path) as owned:
|
||||
yield owned
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _drained(request: pytest.FixtureRequest) -> None:
|
||||
for name in ("wire", "sink"):
|
||||
if name in request.fixturenames:
|
||||
request.getfixturevalue(name).drain()
|
||||
|
||||
|
||||
def _routes(received: Sequence[Request]) -> list[tuple[str, str]]:
|
||||
return [(request.method, urlsplit(unquote(request.target)).path) for request in received]
|
||||
|
||||
|
||||
def _logged_inputs(sink: Wire, call_ids: Sequence[str]) -> dict[str, JsonValue]:
|
||||
eventually(sink.received.qsize, lambda pending: pending >= len(call_ids), seconds=30)
|
||||
events: Final = tuple(_JSON_OBJECT.validate_json(request.body) for request in sink.drain())
|
||||
return {str(event["call_id"]): event["input"] for event in events}
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
@pytest.mark.parametrize("route", ("/v1/systemone", "/v1/decisions"))
|
||||
def test_an_allowlisted_remote_image_reaches_clef_as_bytes_while_callbacks_log_the_url_the_caller_sent(
|
||||
allowlisted: Gateway, wire: Wire, sink: Wire, route: Route
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
image_url: Final = f"{wire.url}/image.png?id=marker-{marker}"
|
||||
responses: Final = tuple(_decide(allowlisted, route, _ALLOWLISTED_MODEL, marker, [image_url]) for _ in range(2))
|
||||
for response in responses:
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response(route, "clef", marker)
|
||||
received: Final = wire.drain()
|
||||
posts: Final = tuple(request for request in received if request.method == "POST")
|
||||
assert {target for method, target in _routes(received) if method == "GET"} == {"/image.png"}, _routes(received)
|
||||
assert [_upstream_body(request) for request in posts] == [
|
||||
_expected_upstream_body("clef", _state(marker), [_PNG_DATA_URL])
|
||||
] * 2
|
||||
call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses)
|
||||
assert len(set(call_ids)) == 2, call_ids
|
||||
assert _logged_inputs(sink, call_ids) == {
|
||||
call_id: _expected_upstream_body("clef", _state(marker), [image_url]) for call_id in call_ids
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_a_missing_remote_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = _decide(
|
||||
allowlisted, "/v1/systemone", _ALLOWLISTED_MODEL, marker, [f"{wire.url}/missing.png?id=marker-{marker}"]
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "Unable to fetch image from URL. Status code: 404" in response.text, response.text
|
||||
assert _routes(wire.drain()) == [("GET", "/missing.png")]
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_a_remote_page_that_is_not_an_image_is_refused_without_a_call_to_clef(allowlisted: Gateway, wire: Wire) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = _decide(
|
||||
allowlisted, "/v1/decisions", _ALLOWLISTED_MODEL, marker, [f"{wire.url}/page.html?id=marker-{marker}"]
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "images, got text/html" in response.text, response.text
|
||||
assert _routes(wire.drain()) == [("GET", "/page.html")]
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_a_hanging_image_host_gets_one_attempt_within_the_callers_timeout(allowlisted: Gateway, wire: Wire) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = _decide(
|
||||
allowlisted,
|
||||
"/v1/systemone",
|
||||
_ALLOWLISTED_MODEL,
|
||||
marker,
|
||||
[f"{wire.url}/slow.png?id=marker-{marker}"],
|
||||
timeout=2,
|
||||
num_retries=0,
|
||||
)
|
||||
received: Final = wire.drain()
|
||||
_SLOW_IMAGE_RELEASE.set()
|
||||
assert response.status_code == 400, response.text
|
||||
assert "Unable to fetch image from URL after 3 attempts" in response.text, response.text
|
||||
assert _routes(received) == [("GET", "/slow.png")]
|
||||
|
|
@ -116,6 +116,7 @@ class _Provider:
|
|||
wraps_result: bool
|
||||
cost_map_key: str | None
|
||||
speaks_openai: bool = False
|
||||
takes_images: bool = False
|
||||
ai_decide: bool = False
|
||||
provider_reported_cost: float | None = None
|
||||
|
||||
|
|
@ -184,6 +185,7 @@ _PROVIDERS: Final = (
|
|||
_API_KEY,
|
||||
True,
|
||||
"cloudflare/@cf/cloudflare/clef",
|
||||
takes_images=True,
|
||||
),
|
||||
_Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None),
|
||||
_Provider(
|
||||
|
|
@ -211,9 +213,11 @@ _PROVIDERS: Final = (
|
|||
_Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True),
|
||||
)
|
||||
_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai)
|
||||
_TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images)
|
||||
_PERPLEXITY: Final = _PROVIDERS[0]
|
||||
_TYPESAFE: Final = _PROVIDERS[1]
|
||||
_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai)
|
||||
_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images)
|
||||
_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"}
|
||||
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
|
||||
("missing questions", {"input": _INPUT}),
|
||||
|
|
@ -235,12 +239,13 @@ _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
|
|||
("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}),
|
||||
("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}),
|
||||
)
|
||||
_PNG_DATA_URL: Final = "data:image/png;base64,iVBORw0KGgo="
|
||||
_IMAGE_INPUT: Final[list[JsonValue]] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": _INPUT},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "auto"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
|
@ -250,7 +255,7 @@ _OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [
|
|||
"type": "message",
|
||||
"content": [
|
||||
{"type": "input_text", "text": _INPUT},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "auto"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
|
@ -337,6 +342,13 @@ def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: d
|
|||
assert "Invalid Decisions request" in response.text, (label, response.text)
|
||||
|
||||
|
||||
def _decide_with_images_in_both_formats(gateway: Gateway, model: str) -> tuple[httpx.Response, httpx.Response]:
|
||||
return (
|
||||
_decide(gateway, model, input=_IMAGE_INPUT),
|
||||
_decide_in_system_one_format(gateway, model, images=[_PNG_DATA_URL]),
|
||||
)
|
||||
|
||||
|
||||
def _assert_refused_for_safety_identifier(response: httpx.Response) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
assert "safety_identifier" in response.text and "drop_params" in response.text, response.text
|
||||
|
|
@ -462,18 +474,37 @@ def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gate
|
|||
assert _upstream_calls(gateway, handle) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id)
|
||||
def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call(
|
||||
@pytest.mark.parametrize("provider", _TEXT_ONLY_PROVIDERS, ids=_provider_id)
|
||||
def test_text_only_providers_refuse_images_unless_the_deployment_drops_params(
|
||||
gateway: Gateway, provider: _Provider
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, provider.upstream_reply())
|
||||
model: Final = _deployment(scenario, handle, provider)
|
||||
response: Final = _decide(gateway, model, input=_IMAGE_INPUT)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "input_image" in response.text
|
||||
strict: Final = _deployment(scenario, handle, provider)
|
||||
openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, strict)
|
||||
assert (openai_format.status_code, system_one_format.status_code) == (400, 400), openai_format.text
|
||||
assert "['input_image']" in openai_format.text and "drop_params" in openai_format.text, openai_format.text
|
||||
assert "['images']" in system_one_format.text, system_one_format.text
|
||||
assert _upstream_calls(gateway, handle) == []
|
||||
|
||||
dropping: Final = _deployment(scenario, handle, provider, drop_params=True)
|
||||
accepted: Final = _decide_with_images_in_both_formats(gateway, dropping)
|
||||
assert [response.status_code for response in accepted] == [200, 200], [response.text for response in accepted]
|
||||
assert [call["body"] for call in _upstream_calls(gateway, handle)] == [provider.upstream_body()] * 2
|
||||
|
||||
|
||||
def test_cloudflare_receives_images_from_either_format_in_its_images_field(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, _CLOUDFLARE.upstream_reply())
|
||||
model: Final = _deployment(scenario, handle, _CLOUDFLARE)
|
||||
openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model)
|
||||
assert openai_format.status_code == 200, openai_format.text
|
||||
assert openai_format.json() == _CLOUDFLARE.litellm_response()
|
||||
assert system_one_format.status_code == 200, system_one_format.text
|
||||
assert system_one_format.json()["answers"] == _SYSTEM_ONE_ANSWERS
|
||||
expected_body: Final = {**_CLOUDFLARE.upstream_body(), "images": [_PNG_DATA_URL]}
|
||||
assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2
|
||||
|
||||
|
||||
def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import wait
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.cost_calculator import get_response_cost_from_hidden_params
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.decisions import (
|
||||
ChoiceAnswer,
|
||||
DecisionsResponse,
|
||||
|
|
@ -1539,20 +1541,16 @@ async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_ans
|
|||
("model", "request_kwargs", "message"),
|
||||
(
|
||||
(
|
||||
"typesafe/jev-1.13",
|
||||
"openai/gpt-6-luna",
|
||||
{
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}],
|
||||
}
|
||||
],
|
||||
"input": "review",
|
||||
"images": ["data:image/png;base64,AA=="],
|
||||
"questions": [{"type": "predicate", "instructions": "Is this a defect?"}],
|
||||
},
|
||||
"cannot serve this request",
|
||||
"send images as input_image content parts",
|
||||
),
|
||||
),
|
||||
ids=("image_to_systemone_provider",),
|
||||
ids=("images_and_input",),
|
||||
)
|
||||
async def test_requests_a_provider_cannot_serve_are_rejected_before_http(
|
||||
respx_mock: respx.MockRouter,
|
||||
|
|
@ -1735,6 +1733,776 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_
|
|||
}
|
||||
|
||||
|
||||
_CLOUDFLARE_RUN_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare"
|
||||
_PERPLEXITY_DECISIONS_URL: Final = "https://api.perplexity.ai/v1/decisions"
|
||||
_PNG_DATA_URL: Final = "data:image/png;base64,AA=="
|
||||
_REMOTE_IMAGE_URL: Final = "https://images.example/screen.png"
|
||||
_SCREEN_TEXT: Final = "Is the screen cracked?"
|
||||
_SYSTEMONE_PREDICATE: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}
|
||||
)
|
||||
|
||||
|
||||
def _screen_messages(image_url: str) -> tuple[Mapping[str, object], ...]:
|
||||
return (
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": _SCREEN_TEXT},
|
||||
{"type": "input_image", "image_url": image_url, "detail": "high"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _image_requests(image_url: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
|
||||
return (
|
||||
("decisions", MappingProxyType({"input": _screen_messages(image_url), "questions": _PREDICATE_QUESTIONS})),
|
||||
(
|
||||
"systemone",
|
||||
MappingProxyType({"state": _SCREEN_TEXT, "questions": _SYSTEMONE_PREDICATE, "images": [image_url]}),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cloudflare_account(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("model", ("clef", "clef-flash"))
|
||||
async def test_input_images_reach_cloudflare_clef_as_its_images_field(model: str, respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/{model}").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=f"cloudflare/{model}",
|
||||
input=(
|
||||
*_screen_messages(_PNG_DATA_URL),
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/webp;base64,BB=="}]},
|
||||
{"role": "user", "content": "Order 1234."},
|
||||
),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": model,
|
||||
"state": f"{_SCREEN_TEXT}\n\nOrder 1234.",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL, "data:image/webp;base64,BB=="],
|
||||
}
|
||||
assert isinstance(response, OpenAIDecisionResponse)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_systemone_images_reach_cloudflare_clef_in_the_callers_form(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state={"ticket": 1234},
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL, {"content_type": "image/webp", "base64": "BB=="}],
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "clef",
|
||||
"state": {"ticket": 1234},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL, {"content_type": "image/webp", "base64": "BB=="}],
|
||||
}
|
||||
assert isinstance(response, DecisionsResponse)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class _InputRecordingLogger(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.inputs: list[object] = []
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.inputs.append(kwargs.get("input"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS)
|
||||
async def test_logging_callbacks_get_the_image_url_the_caller_sent_while_cloudflare_gets_its_bytes(
|
||||
api: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
async_only_image_fetch,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
recording_logger: Final = _InputRecordingLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recording_logger])
|
||||
|
||||
await getattr(litellm, f"a{api}")(model="cloudflare/clef", **request_kwargs)
|
||||
await _drain_logging_worker()
|
||||
|
||||
wire_body: Final = json.loads(route.calls[0].request.content)
|
||||
assert wire_body["images"] == [async_only_image_fetch.data_url]
|
||||
assert recording_logger.inputs == [{**wire_body, "images": [_REMOTE_IMAGE_URL]}]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS)
|
||||
async def test_remote_images_are_fetched_off_the_event_loop_and_inlined_for_cloudflare(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, async_only_image_fetch
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await getattr(litellm, f"a{api}")(model="cloudflare/clef", **request_kwargs)
|
||||
|
||||
assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL]
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [async_only_image_fetch.data_url]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS)
|
||||
def test_sync_decisions_fetch_and_inline_remote_images_for_cloudflare(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
png: Final = b"\x89PNG\r\n\x1a\n"
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(content=png, headers={"content-type": "image/png"})
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
getattr(litellm, api)(model="cloudflare/clef", **request_kwargs)
|
||||
|
||||
assert image_route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [
|
||||
f"data:image/png;base64,{base64.b64encode(png).decode()}"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("sync", (False, True), ids=("async", "sync"))
|
||||
async def test_remote_image_downloads_use_the_callers_timeout(
|
||||
sync: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()))
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(
|
||||
content=b"\x89PNG\r\n\x1a\n", headers={"content-type": "image/png"}
|
||||
)
|
||||
respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
request_kwargs: Final = {**_image_requests(_REMOTE_IMAGE_URL)[1][1], "model": "cloudflare/clef", "timeout": 7.0}
|
||||
|
||||
if sync:
|
||||
litellm.systemone(**request_kwargs)
|
||||
else:
|
||||
await litellm.asystemone(**request_kwargs)
|
||||
|
||||
sent_timeout: Final = image_route.calls.last.request.extensions["timeout"]
|
||||
assert sent_timeout.keys() == httpx.Timeout(7.0).as_dict().keys()
|
||||
assert all(6.0 < seconds <= 7.0 for seconds in sent_timeout.values()), sent_timeout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remote_input_images_reach_openai_unfetched(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, async_only_image_fetch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
input=_screen_messages(_REMOTE_IMAGE_URL),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["input"][0]["content"][1]["image_url"] == _REMOTE_IMAGE_URL
|
||||
assert async_only_image_fetch.fetched == []
|
||||
|
||||
|
||||
_IMAGE_REFUSALS: Final[tuple[tuple[str, Mapping[str, object], str], ...]] = (
|
||||
("decisions", _image_requests(_PNG_DATA_URL)[0][1], "input_image"),
|
||||
("systemone", _image_requests(_PNG_DATA_URL)[1][1], "images"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("api", "request_kwargs", "param"), _IMAGE_REFUSALS, ids=_PREDICATE_REQUEST_IDS)
|
||||
def test_images_are_refused_before_http_when_the_provider_cannot_take_them(
|
||||
api: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
param: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught:
|
||||
getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_PNG_DATA_URL), ids=_PREDICATE_REQUEST_IDS)
|
||||
async def test_images_are_dropped_from_the_wire_under_drop_params(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
await getattr(litellm, f"a{api}")(
|
||||
model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": _SCREEN_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
_MALFORMED_IMAGE_ENTRIES: Final[tuple[Mapping[str, object] | str, ...]] = (
|
||||
{"url": _PNG_DATA_URL},
|
||||
{"type": "image_url", "image_url": {"url": _PNG_DATA_URL}},
|
||||
{"content_type": "image/png", "base64": "AA==", "detail": "high"},
|
||||
{"content_type": "text/plain", "base64": "AA=="},
|
||||
{"content_type": "image/png", "base64": ""},
|
||||
"a" * 5000,
|
||||
"data:text/plain;base64,AA==",
|
||||
)
|
||||
_MALFORMED_IMAGE_ENTRY_IDS: Final = (
|
||||
"url_object",
|
||||
"chat_image_url",
|
||||
"extra_key",
|
||||
"non_image_type",
|
||||
"empty_base64",
|
||||
"not_a_url",
|
||||
"non_image_data_url",
|
||||
)
|
||||
_MALFORMED_IMAGES: Final[tuple[object, ...]] = (_PNG_DATA_URL, *([entry] for entry in _MALFORMED_IMAGE_ENTRIES))
|
||||
_MALFORMED_IMAGE_IDS: Final = ("bare_string", *_MALFORMED_IMAGE_ENTRY_IDS)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("images", _MALFORMED_IMAGES, ids=_MALFORMED_IMAGE_IDS)
|
||||
def test_malformed_images_are_rejected_before_http(
|
||||
images: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*images"):
|
||||
litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("images", _MALFORMED_IMAGES, ids=_MALFORMED_IMAGE_IDS)
|
||||
async def test_malformed_images_are_dropped_from_the_wire_under_drop_params(
|
||||
images: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images, drop_params=True
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "clef",
|
||||
"state": _SCREEN_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("entry", _MALFORMED_IMAGE_ENTRIES, ids=_MALFORMED_IMAGE_ENTRY_IDS)
|
||||
async def test_a_malformed_image_under_drop_params_keeps_the_well_formed_ones(
|
||||
entry: Mapping[str, object] | str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[entry, _PNG_DATA_URL, {"content_type": "image/png", "base64": "AA=="}],
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [
|
||||
_PNG_DATA_URL,
|
||||
{"content_type": "image/png", "base64": "AA=="},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_images_next_to_input_are_ignored(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
api_key="caller-key",
|
||||
input="review",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
images=[],
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "gpt-6-luna",
|
||||
"input": "review",
|
||||
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_images_next_to_input_are_dropped_under_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
api_key="caller-key",
|
||||
input="review",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
images=[_PNG_DATA_URL],
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "gpt-6-luna",
|
||||
"input": "review",
|
||||
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
}
|
||||
|
||||
|
||||
def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, object]], ...]:
|
||||
return (
|
||||
(
|
||||
"decisions",
|
||||
MappingProxyType(
|
||||
{
|
||||
"input": (
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": _PNG_DATA_URL} for _ in range(count)],
|
||||
},
|
||||
),
|
||||
"questions": _PREDICATE_QUESTIONS,
|
||||
}
|
||||
),
|
||||
),
|
||||
(
|
||||
"systemone",
|
||||
MappingProxyType(
|
||||
{"state": _SCREEN_TEXT, "questions": _SYSTEMONE_PREDICATE, "images": [_PNG_DATA_URL] * count}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _cloudflare_image_requests(5), ids=_PREDICATE_REQUEST_IDS)
|
||||
def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught:
|
||||
getattr(litellm, api)(model="cloudflare/clef", **request_kwargs)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _cloudflare_image_requests(4), ids=_PREDICATE_REQUEST_IDS)
|
||||
def test_the_four_images_cloudflare_takes_reach_it(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
getattr(litellm, api)(model="cloudflare/clef", **request_kwargs)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [_PNG_DATA_URL] * 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dropping_the_only_input_images_is_rejected_before_http(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"\['input_image'\].*leaves no input") as caught:
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
api_key="caller-key",
|
||||
input=[{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
_MALFORMED_IMAGE_URLS: Final = ("not-an-image", "data:application/pdf;base64,AA==", "ftp://images.example/a.png")
|
||||
_MALFORMED_IMAGE_URL_IDS: Final = ("not_a_url", "pdf_data_url", "ftp_url")
|
||||
_OPENAI_DECISIONS_URL: Final = "https://api.openai.com/v1/decisions"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_default_base(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account", "openai_default_base")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "upstream"),
|
||||
(("cloudflare/clef", f"{_CLOUDFLARE_RUN_URL}/clef"), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)),
|
||||
ids=("clef", "luna"),
|
||||
)
|
||||
@pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS)
|
||||
def test_malformed_input_image_urls_are_rejected_before_http(
|
||||
model: str, upstream: str, image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(upstream).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 1 input_image") as caught:
|
||||
litellm.decisions(
|
||||
model=model, api_key="caller-key", input=_screen_messages(image_url), questions=_PREDICATE_QUESTIONS
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS)
|
||||
async def test_a_malformed_input_image_under_drop_params_keeps_the_text_and_well_formed_images(
|
||||
image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=(
|
||||
*_screen_messages(image_url),
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": image_url}]},
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]},
|
||||
),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "clef",
|
||||
"state": _SCREEN_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_dropping_the_only_malformed_input_images_is_rejected_before_http(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 2 input_image") as caught:
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=[
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": url}]}
|
||||
for url in _MALFORMED_IMAGE_URLS[:2]
|
||||
],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
_MIB: Final = 1024 * 1024
|
||||
|
||||
|
||||
def _png_data_url(decoded_bytes: int) -> str:
|
||||
return f"data:image/png;base64,{base64.b64encode(bytes(decoded_bytes)).decode()}"
|
||||
|
||||
|
||||
def _png_object(decoded_bytes: int) -> Mapping[str, object]:
|
||||
return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()}
|
||||
|
||||
|
||||
_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = (
|
||||
((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
(
|
||||
(_png_data_url(4 * _MIB), _png_object(4 * _MIB), _png_data_url(1)),
|
||||
"at most 8,388,608 bytes (8 MiB) of images per request, got 8,388,609 bytes",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params"))
|
||||
@pytest.mark.parametrize(
|
||||
("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib")
|
||||
)
|
||||
def test_images_over_cloudflares_size_limits_are_refused_before_http(
|
||||
images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught:
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=images,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
at_limits: Final = (_png_data_url(4 * _MIB), _png_object(4 * _MIB))
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=[{"role": "user", "content": [{"type": "input_image", "image_url": _png_data_url(4 * _MIB)}]}],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=at_limits
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [_png_data_url(4 * _MIB)]
|
||||
assert json.loads(route.calls[1].request.content)["images"] == list(at_limits)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_a_downloaded_image_over_cloudflares_limit_is_refused_before_the_cloudflare_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(
|
||||
content=bytes(4 * _MIB + 1), headers={"content-type": "image/png"}
|
||||
)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="got one of 4,194,305 bytes") as caught:
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_REMOTE_IMAGE_URL]
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert image_route.call_count == 1
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_remote_image_urls_with_an_uppercase_scheme_are_inlined_for_cloudflare(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
png: Final = b"\x89PNG\r\n\x1a\n"
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(content=png, headers={"content-type": "image/png"})
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_REMOTE_IMAGE_URL.replace("https://", "HTTPS://")],
|
||||
)
|
||||
|
||||
assert image_route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [
|
||||
f"data:image/png;base64,{base64.b64encode(png).decode()}"
|
||||
]
|
||||
|
||||
|
||||
_PNG_DATA_URL_WITH_NAME: Final = "data:image/png;name=screen.png;base64,AA=="
|
||||
_GIF_DATA_URL: Final = "data:image/gif;base64,R0lGODlh"
|
||||
_CLOUDFLARE_TYPE_REFUSAL: Final = "cloudflare takes only image/jpeg, image/jpg, image/png, image/webp images, got "
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("openai_default_base")
|
||||
def test_a_data_url_with_media_type_parameters_reaches_openai_unchanged(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
litellm.decisions(
|
||||
model="openai/gpt-6-luna",
|
||||
api_key="caller-key",
|
||||
input=_screen_messages(_PNG_DATA_URL_WITH_NAME),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["input"][0]["content"][1]["image_url"] == _PNG_DATA_URL_WITH_NAME
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_cloudflare_gets_images_without_media_type_parameters(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[
|
||||
_PNG_DATA_URL_WITH_NAME.replace("image/png", "IMAGE/PNG"),
|
||||
{"content_type": "image/webp;name=screen.webp", "base64": "BB=="},
|
||||
],
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [
|
||||
_PNG_DATA_URL,
|
||||
{"content_type": "image/webp", "base64": "BB=="},
|
||||
]
|
||||
|
||||
|
||||
_REFUSED_TYPE_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = (
|
||||
("decisions", {"input": _screen_messages(_GIF_DATA_URL), "questions": _PREDICATE_QUESTIONS}),
|
||||
(
|
||||
"systemone",
|
||||
{
|
||||
"state": _SCREEN_TEXT,
|
||||
"questions": _SYSTEMONE_PREDICATE,
|
||||
"images": [{"content_type": "image/gif", "base64": "R0lGODlh"}],
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _REFUSED_TYPE_REQUESTS, ids=("input_image", "images_object"))
|
||||
def test_an_image_type_cloudflare_refuses_is_rejected_before_http(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}image/gif")) as caught:
|
||||
getattr(litellm, api)(model="cloudflare/clef", **request_kwargs)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_an_image_type_cloudflare_refuses_is_dropped_under_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=(
|
||||
*_screen_messages(_GIF_DATA_URL),
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]},
|
||||
),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=True,
|
||||
)
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}image/gif")):
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=[{"role": "user", "content": [{"type": "input_image", "image_url": _GIF_DATA_URL}]}],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [_PNG_DATA_URL]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_an_image_dropped_under_drop_params_does_not_count_toward_the_cloudflare_limit(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
gif_then_four_pngs: Final = (
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": _SCREEN_TEXT},
|
||||
*({"type": "input_image", "image_url": url} for url in (_GIF_DATA_URL, *(_PNG_DATA_URL,) * 4)),
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
litellm.decisions(
|
||||
model="cloudflare/clef", input=gif_then_four_pngs, questions=_PREDICATE_QUESTIONS, drop_params=True
|
||||
)
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef", input=gif_then_four_pngs, questions=_PREDICATE_QUESTIONS, drop_params=True
|
||||
)
|
||||
|
||||
assert [json.loads(call.request.content)["images"] for call in route.calls] == [[_PNG_DATA_URL] * 4] * 2
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_a_downloaded_page_that_is_not_an_image_is_refused_before_the_cloudflare_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(
|
||||
content=b"<html></html>", headers={"content-type": "text/html; charset=utf-8"}
|
||||
)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"{_CLOUDFLARE_TYPE_REFUSAL}text/html")) as caught:
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_REMOTE_IMAGE_URL]
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert image_route.call_count == 1
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
|
|
@ -1973,7 +2741,9 @@ async def test_clips_are_dropped_from_the_wire_for_a_model_without_media_support
|
|||
call: Final = getattr(litellm, f"a{api}")
|
||||
|
||||
await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs)
|
||||
await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips)
|
||||
await call(
|
||||
model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips
|
||||
)
|
||||
|
||||
assert len(route.calls) == 2
|
||||
text_only_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
|
@ -2099,7 +2869,12 @@ async def test_media_support_is_read_per_kind_from_the_cost_map(
|
|||
(
|
||||
"typesafe/jev-chatty",
|
||||
"https://example.com/v1/systemone",
|
||||
{"litellm_provider": "typesafe", "mode": "chat", "supports_audio_input": True, "supports_video_input": True},
|
||||
{
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "chat",
|
||||
"supports_audio_input": True,
|
||||
"supports_video_input": True,
|
||||
},
|
||||
),
|
||||
("openai/gpt-audio", "https://example.com/v1/decisions", None),
|
||||
),
|
||||
|
|
@ -2121,7 +2896,9 @@ async def test_a_chat_models_media_flags_do_not_count_for_decisions(
|
|||
audio: Final = ["data:audio/wav;base64,YQ=="]
|
||||
videos: Final = ["data:video/mp4;base64,YQ=="]
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio', 'videos']")):
|
||||
with pytest.raises(
|
||||
litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio', 'videos']")
|
||||
):
|
||||
await litellm.asystemone(
|
||||
model=model,
|
||||
state="test",
|
||||
|
|
|
|||
|
|
@ -3,13 +3,11 @@ import base64
|
|||
import copy
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from httpx import Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm import constants
|
||||
from litellm.litellm_core_utils.prompt_templates import image_handling
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
MAX_CONCURRENT_REMOTE_MEDIA_FETCHES,
|
||||
|
|
@ -275,9 +273,7 @@ def _stub_image_client(content, content_type):
|
|||
|
||||
|
||||
def test_convert_url_to_base64_infers_the_type_when_the_server_sends_octet_stream(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "application/octet-stream")
|
||||
)
|
||||
monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "application/octet-stream"))
|
||||
|
||||
result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}")
|
||||
|
||||
|
|
@ -285,9 +281,7 @@ def test_convert_url_to_base64_infers_the_type_when_the_server_sends_octet_strea
|
|||
|
||||
|
||||
def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "image/jpeg")
|
||||
)
|
||||
monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "image/jpeg"))
|
||||
|
||||
result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}.png")
|
||||
|
||||
|
|
@ -312,7 +306,7 @@ def test_image_size_limit_disabled(monkeypatch):
|
|||
"""
|
||||
Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads.
|
||||
"""
|
||||
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
|
||||
from litellm.litellm_core_utils.prompt_templates import image_handling
|
||||
|
||||
monkeypatch.setattr(litellm, "module_level_client", SmallImageClient())
|
||||
monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0)
|
||||
|
|
@ -580,6 +574,54 @@ def test_convert_url_to_base64_hides_the_ssrf_verdict_and_does_not_retry(monkeyp
|
|||
_assert_verdict_free_messages(messages, url)
|
||||
|
||||
|
||||
def test_convert_url_to_base64_retries_within_one_deadline(monkeypatch):
|
||||
timeouts = []
|
||||
|
||||
def time_out(client, url, timeout=None, **kwargs):
|
||||
timeouts.append(timeout)
|
||||
time.sleep(timeout)
|
||||
raise litellm.Timeout(message="timed out", model="", llm_provider="")
|
||||
|
||||
monkeypatch.setattr(image_handling, "safe_get", time_out)
|
||||
started = time.monotonic()
|
||||
with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"):
|
||||
convert_url_to_base64(f"https://slow.example/{uuid.uuid4()}.png", timeout=0.3)
|
||||
|
||||
assert time.monotonic() - started < 0.6
|
||||
assert len(timeouts) == 1 and 0 < timeouts[0] <= 0.3
|
||||
|
||||
|
||||
async def test_async_convert_url_to_base64_retries_within_one_deadline(monkeypatch):
|
||||
timeouts = []
|
||||
|
||||
async def time_out(client, url, timeout=None, **kwargs):
|
||||
timeouts.append(timeout)
|
||||
await asyncio.sleep(timeout)
|
||||
raise litellm.Timeout(message="timed out", model="", llm_provider="")
|
||||
|
||||
monkeypatch.setattr(image_handling, "async_safe_get", time_out)
|
||||
started = time.monotonic()
|
||||
with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"):
|
||||
await async_convert_url_to_base64(f"https://slow.example/{uuid.uuid4()}.png", timeout=0.3)
|
||||
|
||||
assert time.monotonic() - started < 0.6
|
||||
assert len(timeouts) == 1 and 0 < timeouts[0] <= 0.3
|
||||
|
||||
|
||||
def test_convert_url_to_base64_still_retries_fast_failures_three_times(monkeypatch):
|
||||
attempts = []
|
||||
|
||||
def refuse(client, url, timeout=None, **kwargs):
|
||||
attempts.append(timeout)
|
||||
raise ConnectionError("refused")
|
||||
|
||||
monkeypatch.setattr(image_handling, "safe_get", refuse)
|
||||
with pytest.raises(litellm.ImageFetchError, match="after 3 attempts"):
|
||||
convert_url_to_base64(f"https://down.example/{uuid.uuid4()}.png", timeout=30.0)
|
||||
|
||||
assert len(attempts) == 3 and all(0 < timeout <= 30.0 for timeout in attempts)
|
||||
|
||||
|
||||
async def test_async_inline_remote_media_caps_in_flight_fetches_per_request(monkeypatch):
|
||||
in_flight = {"now": 0, "peak": 0}
|
||||
|
||||
|
|
|
|||
|
|
@ -94,11 +94,10 @@ def test_openai_messages_become_systemone_state_text() -> None:
|
|||
|
||||
body: Final = ir_to_systemone_request("jev-latest", ir)
|
||||
|
||||
assert isinstance(body, Mapping)
|
||||
assert body["state"] == "The package arrived broken.\n\nI want a refund.\n\nOrder 1234."
|
||||
|
||||
|
||||
def test_image_input_is_unsupported_by_systemone_providers() -> None:
|
||||
def test_openai_input_images_reach_a_systemone_provider_as_its_images_in_message_order() -> None:
|
||||
ir: Final = _openai_ir(
|
||||
{
|
||||
"input": [
|
||||
|
|
@ -106,15 +105,27 @@ def test_image_input_is_unsupported_by_systemone_providers() -> None:
|
|||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Is the screen cracked?"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AA=="},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "high"},
|
||||
],
|
||||
}
|
||||
},
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/webp;base64,BB=="}]},
|
||||
{"role": "user", "content": "Order 1234."},
|
||||
],
|
||||
"questions": [{"type": "predicate", "instructions": "Is this a defect?"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(ir_to_systemone_request("jev-latest", ir), UnsupportedDecisionsRequest)
|
||||
body: Final = ir_to_systemone_request("clef", ir)
|
||||
|
||||
assert body["state"] == "Is the screen cracked?\n\nOrder 1234."
|
||||
assert body["images"] == ["data:image/png;base64,AA==", "data:image/webp;base64,BB=="]
|
||||
|
||||
|
||||
def test_systemone_images_reach_a_systemone_provider_in_the_callers_form() -> None:
|
||||
images: Final = ["data:image/png;base64,AA==", {"content_type": "image/jpeg", "base64": "BB=="}]
|
||||
ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "images": images}))
|
||||
|
||||
assert ir_to_systemone_request("clef", ir) == {"model": "clef", **_SYSTEMONE_REQUEST, "images": images}
|
||||
|
||||
|
||||
def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None:
|
||||
|
|
|
|||
|
|
@ -673,7 +673,7 @@ def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch
|
|||
async def test_async_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch):
|
||||
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
|
||||
|
||||
async def fake_convert(url):
|
||||
async def fake_convert(url, timeout=None):
|
||||
return f"data:image/png;base64,{url}"
|
||||
|
||||
monkeypatch.setattr(image_handling, "async_convert_url_to_base64", fake_convert)
|
||||
|
|
|
|||
|
|
@ -181,6 +181,32 @@ def test_systemone_questions_without_instructions_become_valid_openai_questions(
|
|||
assert all(question.instructions for question in _OPENAI_BODY.validate_python(body).questions)
|
||||
|
||||
|
||||
def test_systemone_images_reach_openai_as_input_images_ahead_of_the_state() -> None:
|
||||
request: Final = _SYSTEMONE_BODY.validate_python(
|
||||
{
|
||||
"state": {"ticket": 1234},
|
||||
"questions": {"damaged": {"type": "noul", "instructions": "Is the item damaged?"}},
|
||||
"images": ["data:image/png;base64,AA==", {"content_type": "image/webp", "base64": "BB=="}],
|
||||
}
|
||||
)
|
||||
|
||||
assert ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) == {
|
||||
"model": "gpt-6-luna",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AA=="},
|
||||
{"type": "input_image", "image_url": "data:image/webp;base64,BB=="},
|
||||
{"type": "input_text", "text": '{"ticket": 1234}'},
|
||||
],
|
||||
}
|
||||
],
|
||||
"questions": [{"type": "predicate", "name": "damaged", "instructions": "Is the item damaged?"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"question",
|
||||
({"type": "choice", "criteria": {"refund": None}}, {"type": "score", "criteria": ["minor"]}),
|
||||
|
|
|
|||
|
|
@ -520,15 +520,8 @@ def test_decisions_return_guardrail_information_when_requested(
|
|||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"model": "decider",
|
||||
"input": [
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}
|
||||
],
|
||||
"questions": [{"type": "predicate", "instructions": "Is this a defect?"}],
|
||||
},
|
||||
),
|
||||
ids=("systemone_body", "colliding_choice_values", "image_input"),
|
||||
ids=("systemone_body", "colliding_choice_values"),
|
||||
)
|
||||
def test_openai_format_decisions_rejects_bodies_it_cannot_translate(
|
||||
client: TestClient,
|
||||
|
|
@ -804,6 +797,158 @@ def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_p
|
|||
}
|
||||
|
||||
|
||||
_IMAGE_TEXT: Final = "Is the screen cracked?"
|
||||
_PNG_DATA_URL: Final = "data:image/png;base64,AA=="
|
||||
_IMAGE_ENDPOINTS: Final = ("/v1/decisions", "/v1/systemone")
|
||||
_IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = (
|
||||
{
|
||||
"model": "decider",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": _IMAGE_TEXT},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL},
|
||||
],
|
||||
}
|
||||
],
|
||||
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
},
|
||||
{
|
||||
"model": "decider",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL],
|
||||
},
|
||||
)
|
||||
_IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True))
|
||||
_IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format")
|
||||
|
||||
|
||||
def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None:
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(model_list=[{"model_name": "decider", "litellm_params": dict(litellm_params)}]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("endpoint", "request_body"), _IMAGE_REQUESTS, ids=_IMAGE_REQUEST_IDS)
|
||||
def test_images_reach_a_cloudflare_clef_deployment_as_its_images_field(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
endpoint: str,
|
||||
request_body: Mapping[str, object],
|
||||
) -> None:
|
||||
_route_decider_to(
|
||||
monkeypatch,
|
||||
{
|
||||
"model": "cloudflare/clef",
|
||||
"api_key": "cloudflare-key",
|
||||
"api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
},
|
||||
)
|
||||
upstream: Final = respx_mock.post(
|
||||
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
|
||||
).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=request_body)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "clef",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "request_body", "param"),
|
||||
tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, ("input_image", "images"), strict=True)),
|
||||
ids=_IMAGE_REQUEST_IDS,
|
||||
)
|
||||
def test_images_to_a_text_only_deployment_are_refused_without_drop_params(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
endpoint: str,
|
||||
request_body: Mapping[str, object],
|
||||
param: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=request_body)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert f"['{param}']" in response.json()["error"]["message"]
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("endpoint", "request_body"), _IMAGE_REQUESTS, ids=_IMAGE_REQUEST_IDS)
|
||||
def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
endpoint: str,
|
||||
request_body: Mapping[str, object],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
_route_decider_to(
|
||||
monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True}
|
||||
)
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=request_body)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params"))
|
||||
def test_malformed_images_are_refused_unless_the_deployment_drops_params(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
deployment_drops_params: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
_route_decider_to(
|
||||
monkeypatch,
|
||||
{
|
||||
"model": "cloudflare/clef",
|
||||
"api_key": "cloudflare-key",
|
||||
"api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
"drop_params": deployment_drops_params,
|
||||
},
|
||||
)
|
||||
upstream: Final = respx_mock.post(
|
||||
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
|
||||
).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(
|
||||
"/v1/systemone", json={**_IMAGE_REQUEST_BODIES[1], "images": [{"url": _PNG_DATA_URL}]}
|
||||
)
|
||||
|
||||
if not deployment_drops_params:
|
||||
assert response.status_code == 400, response.text
|
||||
assert "images" in response.json()["error"]["message"]
|
||||
assert not upstream.called
|
||||
return
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "clef",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions"))
|
||||
@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params"))
|
||||
def test_clips_for_a_model_without_media_support_are_refused_unless_the_deployment_drops_params(
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ _SYSTEMONE_RESPONSE: Final[Mapping[str, object]] = {
|
|||
"usage": {"input_tokens": 12, "output_tokens": 1},
|
||||
}
|
||||
_TYPESAFE_URL: Final = "https://api.typesafe.ai/v1/systemone"
|
||||
_CLOUDFLARE_API_BASE: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run"
|
||||
_CLOUDFLARE_CLEF_URL: Final = f"{_CLOUDFLARE_API_BASE}/@cf/cloudflare/clef"
|
||||
_PNG_DATA_URL: Final = "data:image/png;base64,AA=="
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -105,6 +108,25 @@ async def test_decisions_with_state_still_sends_the_systemone_format_and_warns(
|
|||
assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decisions_with_state_still_sends_its_images(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_CLOUDFLARE_CLEF_URL).respond(json=_SYSTEMONE_RESPONSE)
|
||||
|
||||
with pytest.warns(UserWarning, match=r"litellm\.systemone"):
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
state=_STATE,
|
||||
questions=_SYSTEMONE_QUESTIONS,
|
||||
images=[_PNG_DATA_URL],
|
||||
api_key="caller-key",
|
||||
api_base=_CLOUDFLARE_API_BASE,
|
||||
)
|
||||
|
||||
sent: Final = json.loads(respx_mock.calls.last.request.content)
|
||||
assert sent["images"] == [_PNG_DATA_URL]
|
||||
assert route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_systemone_rejects_openai_question_lists_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
|
||||
|
|
|
|||
32
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
32
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -71553,6 +71553,22 @@ export interface operations {
|
|||
*/
|
||||
content_type: string;
|
||||
})[];
|
||||
/**
|
||||
* Images
|
||||
* @description Images as data URLs or content_type plus base64, for providers that look at images
|
||||
*/
|
||||
images?: (string | {
|
||||
/**
|
||||
* Base64
|
||||
* @description Base64-encoded image bytes
|
||||
*/
|
||||
base64: string;
|
||||
/**
|
||||
* Content Type
|
||||
* @description An image/* media type, such as image/png
|
||||
*/
|
||||
content_type: string;
|
||||
})[] | null;
|
||||
/**
|
||||
* Model
|
||||
* @description A decision model from the proxy model_list
|
||||
|
|
@ -80417,6 +80433,22 @@ export interface operations {
|
|||
*/
|
||||
content_type: string;
|
||||
})[];
|
||||
/**
|
||||
* Images
|
||||
* @description Images as data URLs or content_type plus base64, for providers that look at images
|
||||
*/
|
||||
images?: (string | {
|
||||
/**
|
||||
* Base64
|
||||
* @description Base64-encoded image bytes
|
||||
*/
|
||||
base64: string;
|
||||
/**
|
||||
* Content Type
|
||||
* @description An image/* media type, such as image/png
|
||||
*/
|
||||
content_type: string;
|
||||
})[] | null;
|
||||
/**
|
||||
* Model
|
||||
* @description A decision model from the proxy model_list
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue