mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
Merge 6e30309943 into 2b4cb92c8b
This commit is contained in:
Matt Trefilek
GitHub
commit
2125c5fa1a
10 changed files with 641 additions and 74 deletions
|
|
@ -2,6 +2,7 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
|
|
@ -301,7 +302,7 @@ def _unsupported_params(
|
|||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
)
|
||||
refused_image_param: Final = (
|
||||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
None if provider_config.supports_images_in(model, ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
|
|
@ -332,6 +333,7 @@ def _is_not_box_question(question: DecisionsIRQuestion) -> bool:
|
|||
def _without_unsupported_params(
|
||||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
) -> DecisionsIRRequest:
|
||||
|
|
@ -339,7 +341,7 @@ def _without_unsupported_params(
|
|||
ir_request,
|
||||
input=(
|
||||
ir_request.input
|
||||
if provider_config.supports_images_in(ir_request.input, headers)
|
||||
if provider_config.supports_images_in(model, ir_request.input, headers)
|
||||
else with_images_where(ir_request.input, _no_image)
|
||||
),
|
||||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
|
|
@ -400,7 +402,7 @@ def _provider_ir_request(
|
|||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, model, headers, supports_box_questions)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -583,9 +585,9 @@ def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]:
|
|||
return remote_image_urls(call.ir_request.input)
|
||||
|
||||
|
||||
def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall:
|
||||
def _with_images_replaced(call: DecisionsCall, replacements: Mapping[str, str]) -> DecisionsCall:
|
||||
return replace(
|
||||
call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls))
|
||||
call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, replacements))
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -593,7 +595,7 @@ def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
|
|||
remote_urls: Final = _remote_image_urls(call)
|
||||
if not remote_urls:
|
||||
return call
|
||||
return _with_data_urls(
|
||||
return _with_images_replaced(
|
||||
call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls})
|
||||
)
|
||||
|
||||
|
|
@ -603,7 +605,7 @@ async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCal
|
|||
if not remote_urls:
|
||||
return call
|
||||
data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
|
||||
return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
|
||||
return _with_images_replaced(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
|
||||
|
||||
|
||||
def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool:
|
||||
|
|
@ -612,7 +614,7 @@ def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bo
|
|||
|
||||
|
||||
def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
|
||||
media_types: Final = call.provider_config.image_media_types
|
||||
media_types: Final = call.provider_config.image_rules_for(call.model).image_media_types
|
||||
if media_types is None:
|
||||
return call
|
||||
found: Final = (image_media_type(image) for image in input_images(call.ir_request.input))
|
||||
|
|
@ -621,28 +623,41 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
|
|||
)
|
||||
if not unsupported:
|
||||
return call
|
||||
supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(supported):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=supported))
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)),
|
||||
f"takes only {', '.join(sorted(media_types))} images, got {', '.join(unsupported)}",
|
||||
)
|
||||
|
||||
|
||||
def _with_images_kept(
|
||||
decision_input: DecisionsIRState | DecisionsIRMessages, kept: Sequence[bool]
|
||||
) -> DecisionsIRState | DecisionsIRMessages:
|
||||
flags: Final = iter(kept)
|
||||
return with_images_where(decision_input, lambda _image: next(flags))
|
||||
|
||||
|
||||
def _dropped_or_refused(
|
||||
call: DecisionsCall, kept_input: DecisionsIRState | DecisionsIRMessages, refusal: str
|
||||
) -> DecisionsCall:
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=kept_input))
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, "
|
||||
f"got {', '.join(unsupported)}"
|
||||
),
|
||||
message=f"{call.custom_llm_provider} {refusal}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def _checked_image_count(call: DecisionsCall) -> DecisionsCall:
|
||||
max_images: Final = call.provider_config.max_images
|
||||
max_images: Final = call.provider_config.image_rules_for(call.model).max_images
|
||||
image_count: Final = len(input_images(call.ir_request.input))
|
||||
if max_images is None or image_count <= max_images:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
_with_images_kept(call.ir_request.input, tuple(index < max_images for index in range(image_count))),
|
||||
f"takes at most {max_images} images per request, got {image_count}",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -657,11 +672,8 @@ def _byte_limit(limit: int) -> str:
|
|||
return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)"
|
||||
|
||||
|
||||
def _image_size_error(call: DecisionsCall) -> str | None:
|
||||
sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
def _image_size_error(sizes: Sequence[int], per_image: int | None, total: int | None) -> str | None:
|
||||
largest: Final = max(sizes, default=0)
|
||||
per_image: Final = call.provider_config.max_image_bytes
|
||||
total: Final = call.provider_config.max_total_image_bytes
|
||||
if per_image is not None and largest > per_image:
|
||||
return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes"
|
||||
if total is not None and sum(sizes) > total:
|
||||
|
|
@ -669,14 +681,26 @@ def _image_size_error(call: DecisionsCall) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _kept_within_sizes(sizes: Sequence[int], per_image: int | None, total: int | None) -> tuple[bool, ...]:
|
||||
fits: Final = tuple(per_image is None or size <= per_image for size in sizes)
|
||||
spent: Final = accumulate(size if fit else 0 for size, fit in zip(sizes, fits, strict=True))
|
||||
return tuple(
|
||||
fit and (total is None or bytes_so_far <= total) for fit, bytes_so_far in zip(fits, spent, strict=True)
|
||||
)
|
||||
|
||||
|
||||
def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall:
|
||||
error: Final = _image_size_error(call)
|
||||
sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
rules: Final = call.provider_config.image_rules_for(call.model)
|
||||
error: Final = _image_size_error(sizes, rules.max_image_bytes, rules.max_total_image_bytes)
|
||||
if error is None:
|
||||
return call
|
||||
raise litellm.BadRequestError(
|
||||
message=f"{call.custom_llm_provider} {error}",
|
||||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
return _dropped_or_refused(
|
||||
call,
|
||||
_with_images_kept(
|
||||
call.ir_request.input, _kept_within_sizes(sizes, rules.max_image_bytes, rules.max_total_image_bytes)
|
||||
),
|
||||
error,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -693,6 +717,22 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
|||
return body
|
||||
|
||||
|
||||
def _with_caller_urls(sent: DecisionsCall, supported: DecisionsCall, fetched: DecisionsCall) -> DecisionsCall:
|
||||
pairs: Final = zip(input_images(fetched.ir_request.input), input_images(supported.ir_request.input), strict=True)
|
||||
caller_urls: Final = MappingProxyType(
|
||||
{data_url: url for data_url, url in pairs if isinstance(data_url, str) and isinstance(url, str)}
|
||||
)
|
||||
return _with_images_replaced(sent, caller_urls)
|
||||
|
||||
|
||||
def _logged_body(
|
||||
supported: DecisionsCall, fetched: DecisionsCall, sent: DecisionsCall, body: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
if fetched is supported:
|
||||
return body
|
||||
return _request_body(_with_caller_urls(sent, supported, fetched))
|
||||
|
||||
|
||||
def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
|
||||
returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
if returned is not None and cost_map_key_prices_anything(returned):
|
||||
|
|
@ -756,7 +796,8 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
|
|||
|
||||
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
supported: Final = _checked_image_count(_checked_image_types(call))
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported)))
|
||||
fetched: Final = await _async_with_remote_images_inlined(supported)
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(fetched))
|
||||
body: Final = _request_body(inlined)
|
||||
try:
|
||||
response: Final = await _HANDLER.adecisions(
|
||||
|
|
@ -766,7 +807,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon
|
|||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
logged_body=_logged_body(supported, fetched, inlined, body),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
|
|
@ -780,7 +821,8 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon
|
|||
|
||||
def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
supported: Final = _checked_image_count(_checked_image_types(call))
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported)))
|
||||
fetched: Final = _with_remote_images_inlined(supported)
|
||||
inlined: Final = _checked_image_sizes(_checked_image_types(fetched))
|
||||
body: Final = _request_body(inlined)
|
||||
try:
|
||||
response: Final = _HANDLER.decisions(
|
||||
|
|
@ -790,7 +832,7 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
|||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
logged_body=_logged_body(supported, fetched, inlined, body),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from collections.abc import Callable, Mapping, Sequence
|
|||
from dataclasses import replace
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -65,12 +65,20 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter
|
|||
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
|
||||
_REMOTE_URL_PREFIXES: Final = ("http://", "https://")
|
||||
_DATA_URL_PREFIX: Final = "data:"
|
||||
_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/")
|
||||
_MEDIA_TYPE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"image/jpg": "image/jpeg"})
|
||||
|
||||
_StateItem: TypeAlias = str | Mapping[str, object]
|
||||
|
||||
|
||||
def decisions_text(value: DecisionsJSON) -> str:
|
||||
return value if isinstance(value, str) else json.dumps(value)
|
||||
|
||||
|
||||
def standard_base64(value: str) -> str:
|
||||
return value.translate(_URL_SAFE_TO_STANDARD_BASE64)
|
||||
|
||||
|
||||
def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]:
|
||||
names: Final = tuple(question.name for question in questions if question.name is not None)
|
||||
if len(frozenset(names)) == len(questions):
|
||||
|
|
@ -260,6 +268,53 @@ def _systemone_image(image: DecisionsImage) -> str | Mapping[str, object]:
|
|||
return f"{_DATA_URL_PREFIX}{media_type};base64,{image.partition(',')[2]}"
|
||||
|
||||
|
||||
def _standard_data_url(media_type: str, payload: str) -> str:
|
||||
padding: Final = "=" * (-len(payload) % 4)
|
||||
standard_type: Final = _MEDIA_TYPE_ALIASES.get(media_type, media_type)
|
||||
return f"{_DATA_URL_PREFIX}{standard_type};base64,{standard_base64(payload)}{padding}"
|
||||
|
||||
|
||||
def _data_url(image: DecisionsImage) -> str:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return _standard_data_url(_media_type(image.content_type), image.base64)
|
||||
media_type: Final = image_media_type(image)
|
||||
if media_type is None:
|
||||
return image
|
||||
return _standard_data_url(media_type, image.partition(",")[2])
|
||||
|
||||
|
||||
def _image_item(image: DecisionsImage) -> _StateItem:
|
||||
return {"type": "image_url", "image_url": {"url": _data_url(image)}}
|
||||
|
||||
|
||||
def _part_item(part: OpenAIDecisionInputText | OpenAIDecisionInputImage) -> _StateItem:
|
||||
if isinstance(part, OpenAIDecisionInputText):
|
||||
return part.text
|
||||
return _image_item(part.image_url)
|
||||
|
||||
|
||||
def _message_items(message: OpenAIDecisionInputMessage) -> tuple[_StateItem, ...]:
|
||||
if isinstance(message.content, str):
|
||||
return (message.content,)
|
||||
return tuple(map(_part_item, message.content))
|
||||
|
||||
|
||||
def _state_items(state: DecisionsJSON) -> tuple[object, ...]:
|
||||
if isinstance(state, (str, Mapping)):
|
||||
return (state,)
|
||||
return tuple(state)
|
||||
|
||||
|
||||
def _multimodal_state(decision_input: DecisionsIRState | DecisionsIRMessages) -> tuple[object, ...]:
|
||||
match decision_input:
|
||||
case DecisionsIRState():
|
||||
return (*_state_items(decision_input.state), *map(_image_item, decision_input.images))
|
||||
case DecisionsIRMessages():
|
||||
return tuple(chain.from_iterable(map(_message_items, decision_input.messages)))
|
||||
case _:
|
||||
assert_never(decision_input)
|
||||
|
||||
|
||||
def _optional(key: str, value: object) -> Mapping[str, object]:
|
||||
return {} if value is None else {key: value}
|
||||
|
||||
|
|
@ -298,18 +353,23 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
|
||||
|
||||
def ir_to_systemone_request(
|
||||
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
|
||||
model: str,
|
||||
request: DecisionsIRRequest,
|
||||
*,
|
||||
supports_audio: bool = False,
|
||||
supports_video: bool = False,
|
||||
images_in_state: bool = False,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if request.audio and not supports_audio:
|
||||
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
|
||||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
state: Final = _systemone_state(request.input)
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
state_carries_images: Final = images_in_state and bool(images)
|
||||
return {
|
||||
"model": model,
|
||||
"state": state,
|
||||
"state": _multimodal_state(request.input) if state_carries_images else _systemone_state(request.input),
|
||||
**(
|
||||
{"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]}
|
||||
if request.audio
|
||||
|
|
@ -321,7 +381,7 @@ def ir_to_systemone_request(
|
|||
else {}
|
||||
),
|
||||
"questions": {key: _systemone_question(question) for key, question in keyed_questions},
|
||||
**({"images": [_systemone_image(image) for image in images]} if images else {}),
|
||||
**({"images": [_systemone_image(image) for image in images]} if images and not state_carries_images else {}),
|
||||
**(
|
||||
{"assets": {key: asset.model_dump(exclude_none=True) for key, asset in request.assets.items()}}
|
||||
if request.assets
|
||||
|
|
@ -523,6 +583,7 @@ class BaseDecisionsConfig(ABC):
|
|||
max_total_image_bytes: int | None = None
|
||||
image_media_types: frozenset[str] | None = None
|
||||
inlines_remote_images: bool = False
|
||||
images_in_state: bool = False
|
||||
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
|
||||
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
|
||||
)
|
||||
|
|
@ -605,10 +666,13 @@ class BaseDecisionsConfig(ABC):
|
|||
return headers, None
|
||||
|
||||
def supports_images_in(
|
||||
self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
|
||||
self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
|
||||
) -> bool:
|
||||
return self.supports_images
|
||||
|
||||
def image_rules_for(self, model: str) -> "BaseDecisionsConfig":
|
||||
return self
|
||||
|
||||
def supports_assets(self, headers: Mapping[str, str]) -> bool:
|
||||
return False
|
||||
|
||||
|
|
@ -642,6 +706,7 @@ class BaseDecisionsConfig(ABC):
|
|||
request,
|
||||
supports_audio=self.supports_audio_input(model, custom_llm_provider),
|
||||
supports_video=self.supports_video_input(model, custom_llm_provider),
|
||||
images_in_state=self.images_in_state,
|
||||
)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
|
|
|
|||
|
|
@ -1,17 +1,44 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRResponse
|
||||
from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig
|
||||
from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig
|
||||
from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRMessages, DecisionsIRResponse, DecisionsIRState
|
||||
|
||||
_ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType(
|
||||
{
|
||||
"typesafe": TypeSafeDecisionsConfig(),
|
||||
"cloudflare": CloudflareDecisionsConfig(),
|
||||
"perplexity": PerplexityDecisionsConfig(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _upstream_org(model: str) -> str:
|
||||
return model.removeprefix("~").partition("/")[0]
|
||||
|
||||
|
||||
class OpenRouterDecisionsConfig(BaseDecisionsConfig):
|
||||
path = "/alpha/decisions"
|
||||
api_key_env = ("OPENROUTER_API_KEY",)
|
||||
api_base_env = ("OPENROUTER_API_BASE",)
|
||||
inlines_remote_images = True
|
||||
images_in_state = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://openrouter.ai/api"
|
||||
|
||||
def supports_images_in(
|
||||
self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
|
||||
) -> bool:
|
||||
return _upstream_org(model) in _ORG_IMAGE_RULES
|
||||
|
||||
def image_rules_for(self, model: str) -> BaseDecisionsConfig:
|
||||
return _ORG_IMAGE_RULES.get(_upstream_org(model), self)
|
||||
|
||||
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
|
||||
cost: Final = response.usage.extra.get("cost") if response.usage is not None else None
|
||||
if not isinstance(cost, (int, float)):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,11 @@ class PerplexityDecisionsConfig(BaseDecisionsConfig):
|
|||
path = "/v1/decisions"
|
||||
api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY")
|
||||
api_base_env = ("PERPLEXITY_API_BASE",)
|
||||
supports_images = True
|
||||
max_images = 8
|
||||
image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"})
|
||||
inlines_remote_images = True
|
||||
images_in_state = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://api.perplexity.ai"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from types import MappingProxyType
|
|||
from typing import Final, Literal
|
||||
|
||||
from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type, standard_base64
|
||||
from litellm.types.decisions import (
|
||||
DecisionsAsset,
|
||||
DecisionsIRMessages,
|
||||
|
|
@ -22,7 +22,6 @@ _IMAGES_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}images"
|
|||
_ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType(
|
||||
{"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"}
|
||||
)
|
||||
_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/")
|
||||
|
||||
|
||||
def _image_reference(image_url: str) -> str:
|
||||
|
|
@ -36,7 +35,7 @@ def _image_asset(image_url: str) -> DecisionsAsset | None:
|
|||
return DecisionsAsset(
|
||||
type="image",
|
||||
format=asset_format,
|
||||
data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64),
|
||||
data_base64=standard_base64(image_url.partition(",")[2]),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -85,7 +84,7 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig):
|
|||
return "https://api.typesafe.ai"
|
||||
|
||||
def supports_images_in(
|
||||
self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
|
||||
self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str]
|
||||
) -> bool:
|
||||
return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -45868,6 +45868,18 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"openrouter/typesafe/jev-1.14": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "evaluation",
|
||||
"source": "https://openrouter.ai/docs/guides/community/jev",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"openrouter/~typesafe/jev-latest": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "evaluation",
|
||||
"source": "https://openrouter.ai/docs/guides/community/jev",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
|
||||
"input_cost_per_token": 6.7e-07,
|
||||
"litellm_provider": "ovhcloud",
|
||||
|
|
|
|||
|
|
@ -45868,6 +45868,18 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"openrouter/typesafe/jev-1.14": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "evaluation",
|
||||
"source": "https://openrouter.ai/docs/guides/community/jev",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"openrouter/~typesafe/jev-latest": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "evaluation",
|
||||
"source": "https://openrouter.ai/docs/guides/community/jev",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
|
||||
"input_cost_per_token": 6.7e-07,
|
||||
"litellm_provider": "ovhcloud",
|
||||
|
|
|
|||
|
|
@ -162,6 +162,7 @@ _PROVIDERS: Final = (
|
|||
_API_KEY,
|
||||
False,
|
||||
"perplexity/pplx-decider-v1-27b",
|
||||
takes_images=True,
|
||||
),
|
||||
_Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"),
|
||||
_Provider(
|
||||
|
|
@ -172,6 +173,7 @@ _PROVIDERS: Final = (
|
|||
_API_KEY,
|
||||
False,
|
||||
None,
|
||||
takes_images=True,
|
||||
provider_reported_cost=1.5834e-5,
|
||||
),
|
||||
_Provider(
|
||||
|
|
@ -215,9 +217,10 @@ _PROVIDERS: Final = (
|
|||
_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai)
|
||||
_TEXT_ONLY_PROVIDERS: Final = tuple(provider for provider in _SYSTEM_ONE_PROVIDERS if not provider.takes_images)
|
||||
_PERPLEXITY: Final = _PROVIDERS[0]
|
||||
_OPENROUTER: Final = next(provider for provider in _PROVIDERS if provider.name == "openrouter")
|
||||
_TYPESAFE: Final = _PROVIDERS[1]
|
||||
_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai)
|
||||
_CLOUDFLARE: Final = next(provider for provider in _SYSTEM_ONE_PROVIDERS if provider.takes_images)
|
||||
_CLOUDFLARE: Final = next(provider for provider in _PROVIDERS if provider.name == "cloudflare")
|
||||
_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"}
|
||||
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
|
||||
("missing questions", {"input": _INPUT}),
|
||||
|
|
@ -510,6 +513,21 @@ def test_cloudflare_receives_images_from_either_format_in_its_images_field(gatew
|
|||
assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", (_PERPLEXITY, _OPENROUTER), ids=_provider_id)
|
||||
def test_perplexity_and_openrouter_receive_images_from_either_format_in_their_state(
|
||||
gateway: Gateway, provider: _Provider
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, provider.upstream_reply())
|
||||
model: Final = _deployment(scenario, handle, provider)
|
||||
openai_format, system_one_format = _decide_with_images_in_both_formats(gateway, model)
|
||||
assert openai_format.status_code == 200, openai_format.text
|
||||
assert system_one_format.status_code == 200, system_one_format.text
|
||||
image_item: Final[dict[str, JsonValue]] = {"type": "image_url", "image_url": {"url": _PNG_DATA_URL}}
|
||||
expected_body: Final = {**provider.upstream_body(), "state": [_INPUT, image_item]}
|
||||
assert [call["body"] for call in _upstream_calls(gateway, handle)] == [expected_body] * 2
|
||||
|
||||
|
||||
def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, _OPENAI.upstream_reply())
|
||||
|
|
|
|||
|
|
@ -1913,6 +1913,8 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_
|
|||
|
||||
_CLOUDFLARE_RUN_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare"
|
||||
_PERPLEXITY_DECISIONS_URL: Final = "https://api.perplexity.ai/v1/decisions"
|
||||
_TEXT_ONLY_MODEL: Final = "nace/drex-v2.0"
|
||||
_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone"
|
||||
_PNG_DATA_URL: Final = "data:image/png;base64,AA=="
|
||||
_REMOTE_IMAGE_URL: Final = "https://images.example/screen.png"
|
||||
_SCREEN_TEXT: Final = "Is the screen cracked?"
|
||||
|
|
@ -2119,10 +2121,10 @@ def test_images_are_refused_before_http_when_the_provider_cannot_take_them(
|
|||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught:
|
||||
getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs)
|
||||
getattr(litellm, api)(model=_TEXT_ONLY_MODEL, api_key="caller-key", **request_kwargs)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
|
@ -2134,14 +2136,12 @@ async def test_images_are_dropped_from_the_wire_under_drop_params(
|
|||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
|
||||
|
||||
await getattr(litellm, f"a{api}")(
|
||||
model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs
|
||||
)
|
||||
await getattr(litellm, f"a{api}")(model=_TEXT_ONLY_MODEL, api_key="caller-key", drop_params=True, **request_kwargs)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"model": "drex-v2.0",
|
||||
"state": _SCREEN_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
|
@ -2307,7 +2307,7 @@ def _cloudflare_image_requests(count: int) -> tuple[tuple[str, Mapping[str, obje
|
|||
def test_more_images_than_cloudflare_takes_are_refused_before_any_download_or_http(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught:
|
||||
|
|
@ -2334,11 +2334,11 @@ async def test_dropping_the_only_input_images_is_rejected_before_http(
|
|||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
route: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"\['input_image'\].*leaves no input") as caught:
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
model=_TEXT_ONLY_MODEL,
|
||||
api_key="caller-key",
|
||||
input=[{"role": "user", "content": [{"type": "input_image", "image_url": _PNG_DATA_URL}]}],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
|
|
@ -2445,6 +2445,10 @@ def _png_object(decoded_bytes: int) -> Mapping[str, object]:
|
|||
return {"content_type": "image/png", "base64": base64.b64encode(bytes(decoded_bytes)).decode()}
|
||||
|
||||
|
||||
def _distinct_png_data_urls(count: int) -> tuple[str, ...]:
|
||||
return tuple(f"data:image/png;base64,{base64.b64encode(bytes((index,))).decode()}" for index in range(count))
|
||||
|
||||
|
||||
_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = (
|
||||
((_png_data_url(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
((_png_object(4 * _MIB + 1),), "images of at most 4,194,304 bytes (4 MiB) each, got one of 4,194,305 bytes"),
|
||||
|
|
@ -2455,29 +2459,111 @@ _OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], str], ...]] = (
|
|||
)
|
||||
|
||||
|
||||
_OVERSIZED_IMAGE_IDS: Final = ("data_url_over_4mib", "object_over_4mib", "total_over_8mib")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params"))
|
||||
@pytest.mark.parametrize(
|
||||
("images", "refusal"), _OVERSIZED_IMAGES, ids=("data_url_over_4mib", "object_over_4mib", "total_over_8mib")
|
||||
)
|
||||
@pytest.mark.parametrize(("images", "refusal"), _OVERSIZED_IMAGES, ids=_OVERSIZED_IMAGE_IDS)
|
||||
def test_images_over_cloudflares_size_limits_are_refused_before_http(
|
||||
images: Sequence[object], refusal: str, drop_params: bool, respx_mock: respx.MockRouter
|
||||
images: Sequence[object], refusal: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(f"cloudflare takes {refusal}")) as caught:
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=images,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
litellm.systemone(model="cloudflare/clef", state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=images)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
_SMALL_PNG: Final = _png_data_url(1)
|
||||
_DROPPED_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], Sequence[object]], ...]] = (
|
||||
*((images, images[:kept]) for (images, _), kept in zip(_OVERSIZED_IMAGES, (0, 0, 2), strict=True)),
|
||||
((_png_data_url(4 * _MIB + 1), _SMALL_PNG), (_SMALL_PNG,)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(
|
||||
("images", "kept"), _DROPPED_OVERSIZED_IMAGES, ids=(*_OVERSIZED_IMAGE_IDS, "oversized_then_small")
|
||||
)
|
||||
def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params(
|
||||
images: Sequence[object], kept: Sequence[object], respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=images,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content).get("images", []) == list(kept)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_logging_callbacks_get_only_the_images_left_after_an_oversized_one_drops(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
recording_logger: Final = _InputRecordingLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recording_logger])
|
||||
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_png_data_url(4 * _MIB + 1), _SMALL_PNG],
|
||||
drop_params=True,
|
||||
)
|
||||
await _drain_logging_worker()
|
||||
|
||||
assert recording_logger.inputs == [json.loads(route.calls.last.request.content)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_logging_callbacks_get_the_caller_url_of_a_remote_image_kept_after_an_oversized_one_drops(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, async_only_image_fetch
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
recording_logger: Final = _InputRecordingLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recording_logger])
|
||||
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_png_data_url(4 * _MIB + 1), _REMOTE_IMAGE_URL],
|
||||
drop_params=True,
|
||||
)
|
||||
await _drain_logging_worker()
|
||||
|
||||
wire_body: Final = json.loads(route.calls.last.request.content)
|
||||
assert wire_body["images"] == [async_only_image_fetch.data_url]
|
||||
assert recording_logger.inputs == [{**wire_body, "images": [_REMOTE_IMAGE_URL]}]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
def test_images_past_cloudflares_limit_are_dropped_under_drop_params(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
images: Final = _distinct_png_data_urls(5)
|
||||
|
||||
litellm.systemone(
|
||||
model="cloudflare/clef",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=list(images),
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["images"] == list(images[:4])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_images_at_cloudflares_size_limits_reach_it(respx_mock: respx.MockRouter) -> None:
|
||||
|
|
@ -3588,6 +3674,47 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_
|
|||
assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest"))
|
||||
async def test_openrouter_jev_sends_box_questions_and_returns_box_answers(
|
||||
model: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(
|
||||
json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}}
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=model,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}],
|
||||
}
|
||||
],
|
||||
questions=[_CRACK_BOX_QUESTION],
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["questions"] == {
|
||||
"crack": {"type": "box", "instructions": "the crack on the screen"}
|
||||
}
|
||||
assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)]
|
||||
|
||||
|
||||
def test_openrouter_jev_1_13_still_refuses_box_questions(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"):
|
||||
litellm.decisions(
|
||||
model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k"
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_box_questions_off_is_refused_even_when_the_cost_map_allows_them(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -4253,6 +4380,260 @@ async def test_hosted_vllm_decisions_sends_input_images_to_systemone(
|
|||
assert async_only_image_fetch.fetched == (["https://images.example/input.png"] if remote_image else [])
|
||||
|
||||
|
||||
_STATE_IMAGE_PROVIDERS: Final = (
|
||||
("openrouter/cloudflare/clef", "https://openrouter.ai/api/alpha/decisions", "cloudflare/clef"),
|
||||
("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", "pplx-decider-v1.2"),
|
||||
)
|
||||
_STATE_IMAGE_PROVIDER_IDS: Final = ("openrouter", "perplexity")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote"))
|
||||
@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS)
|
||||
async def test_decisions_sends_input_images_in_state_order(
|
||||
model: str,
|
||||
url: str,
|
||||
upstream_model: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
remote_image: bool,
|
||||
async_only_image_fetch: AsyncOnlyImageFetch,
|
||||
) -> None:
|
||||
image: Final[str] = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA=="
|
||||
expected_image: Final[str] = async_only_image_fetch.data_url if remote_image else image
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=model,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Listing photo:"},
|
||||
{"type": "input_image", "image_url": image},
|
||||
{"type": "input_text", "text": "Is it damaged?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content) == {
|
||||
"model": upstream_model,
|
||||
"state": [
|
||||
"Listing photo:",
|
||||
{"type": "image_url", "image_url": {"url": expected_image}},
|
||||
"Is it damaged?",
|
||||
],
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
assert response.answers[0].probability == 0.9
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS)
|
||||
def test_systemone_moves_images_into_state(
|
||||
model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state={"ticket": 1234},
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
images=["data:IMAGE/JPG;name=shot.jpg;base64,A-_AB-", {"content_type": "image/webp", "base64": "B-_B"}],
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content) == {
|
||||
"model": upstream_model,
|
||||
"state": [
|
||||
{"ticket": 1234},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A+/AB+=="}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/webp;base64,B+/B"}},
|
||||
],
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS)
|
||||
def test_a_list_state_keeps_its_items_ahead_of_the_images(
|
||||
model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=["Ticket 1234", {"severity": "high"}],
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL],
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["state"] == [
|
||||
"Ticket 1234",
|
||||
{"severity": "high"},
|
||||
{"type": "image_url", "image_url": {"url": _PNG_DATA_URL}},
|
||||
]
|
||||
|
||||
|
||||
_STATE_IMAGE_LIMITS: Final = (
|
||||
("perplexity/pplx-decider-v1.2", "https://api.perplexity.ai/v1/decisions", 8),
|
||||
("openrouter/cloudflare/clef-flash", "https://openrouter.ai/api/alpha/decisions", 4),
|
||||
("openrouter/perplexity/pplx-decider-v1.2", "https://openrouter.ai/api/alpha/decisions", 8),
|
||||
)
|
||||
_STATE_IMAGE_LIMIT_IDS: Final = ("perplexity", "openrouter-clef", "openrouter-perplexity")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS)
|
||||
def test_more_images_than_the_model_takes_are_refused_before_http(
|
||||
model: str, url: str, max_images: int, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL] * max_images,
|
||||
api_key="k",
|
||||
)
|
||||
with pytest.raises(
|
||||
litellm.BadRequestError,
|
||||
match=re.escape(f"takes at most {max_images} images per request, got {max_images + 1}"),
|
||||
):
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL] * (max_images + 1),
|
||||
api_key="k",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "max_images"), _STATE_IMAGE_LIMITS, ids=_STATE_IMAGE_LIMIT_IDS)
|
||||
def test_images_past_the_models_limit_are_dropped_under_drop_params(
|
||||
model: str, url: str, max_images: int, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
images: Final = _distinct_png_data_urls(max_images + 2)
|
||||
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=list(images),
|
||||
api_key="k",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["state"] == [
|
||||
_SCREEN_TEXT,
|
||||
*({"type": "image_url", "image_url": {"url": image}} for image in images[:max_images]),
|
||||
]
|
||||
|
||||
|
||||
def test_an_image_over_the_clef_size_limit_is_refused_before_the_openrouter_call(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
over_four_mib: Final = "data:image/png;base64," + "A" * (4 * 1024 * 1024 // 3 * 4 + 8)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape("openrouter takes images of at most 4,194,304 bytes")):
|
||||
litellm.systemone(
|
||||
model="openrouter/cloudflare/clef-flash",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[over_four_mib],
|
||||
api_key="k",
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "url", "upstream_model"), _STATE_IMAGE_PROVIDERS, ids=_STATE_IMAGE_PROVIDER_IDS)
|
||||
def test_a_gif_to_a_state_image_provider_is_rejected_before_http(
|
||||
model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(
|
||||
litellm.BadRequestError, match=re.escape("takes only image/jpeg, image/jpg, image/png, image/webp images")
|
||||
) as caught:
|
||||
litellm.systemone(
|
||||
model=model, state=_SCREEN_TEXT, questions=_SYSTEMONE_PREDICATE, images=[_GIF_DATA_URL], api_key="k"
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest"))
|
||||
def test_openrouter_jev_rejects_webp_before_http(
|
||||
model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape("got image/webp")):
|
||||
litellm.systemone(
|
||||
model=model,
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=["data:image/webp;base64,BB=="],
|
||||
api_key="k",
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_images_to_an_openrouter_upstream_without_image_rules_are_refused_or_dropped_as_before(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
request: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
"model": "openrouter/upstage/solar-decide-flash",
|
||||
"state": "Product photo",
|
||||
"questions": _SYSTEMONE_PREDICATE,
|
||||
"images": [_SMALL_PNG],
|
||||
"api_key": "caller-key",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=r"\['images'\]"):
|
||||
await litellm.asystemone(**request)
|
||||
await litellm.asystemone(**request, drop_params=True)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls.last.request.content) == {
|
||||
"model": "upstage/solar-decide-flash",
|
||||
"state": "Product photo",
|
||||
"questions": _SYSTEMONE_PREDICATE,
|
||||
}
|
||||
|
||||
|
||||
def test_openrouter_jev_drops_webp_and_keeps_png_under_drop_params(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
|
||||
litellm.systemone(
|
||||
model="openrouter/typesafe/jev-1.14",
|
||||
state=_SCREEN_TEXT,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=["data:image/webp;base64,BB==", "data:image/png;base64,AA=="],
|
||||
api_key="k",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["state"] == [
|
||||
_SCREEN_TEXT,
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
||||
]
|
||||
|
||||
|
||||
def test_hosted_vllm_systemone_forwards_images(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
|
|
|
|||
|
|
@ -962,6 +962,8 @@ _IMAGE_REQUEST_BODIES: Final[tuple[Mapping[str, object], ...]] = (
|
|||
)
|
||||
_IMAGE_REQUESTS: Final = tuple(zip(_IMAGE_ENDPOINTS, _IMAGE_REQUEST_BODIES, strict=True))
|
||||
_IMAGE_REQUEST_IDS: Final = ("openai_format", "systemone_format")
|
||||
_TEXT_ONLY_DEPLOYMENT: Final[Mapping[str, object]] = {"model": "nace/drex-v2.0", "api_key": "test-key"}
|
||||
_TEXT_ONLY_URL: Final = "https://console.nace.ai/v1/systemone"
|
||||
|
||||
|
||||
def _route_decider_to(monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]) -> None:
|
||||
|
|
@ -1017,7 +1019,8 @@ def test_images_to_a_text_only_deployment_are_refused_without_drop_params(
|
|||
param: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
_route_decider_to(monkeypatch, _TEXT_ONLY_DEPLOYMENT)
|
||||
upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=request_body)
|
||||
|
||||
|
|
@ -1035,16 +1038,14 @@ def test_images_to_a_text_only_deployment_are_dropped_when_it_drops_params(
|
|||
request_body: Mapping[str, object],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
_route_decider_to(
|
||||
monkeypatch, {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": True}
|
||||
)
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
_route_decider_to(monkeypatch, {**_TEXT_ONLY_DEPLOYMENT, "drop_params": True})
|
||||
upstream: Final = respx_mock.post(_TEXT_ONLY_URL).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=request_body)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"model": "drex-v2.0",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
|
@ -1104,7 +1105,12 @@ def test_malformed_assets_for_a_deployment_without_asset_support_are_dropped_onl
|
|||
|
||||
response: Final = client.post(
|
||||
"/v1/systemone",
|
||||
json={**_IMAGE_REQUEST_BODIES[1], "assets": {"photo-1": {"type": "image", "part": "file1"}}},
|
||||
json={
|
||||
"model": "decider",
|
||||
"state": _IMAGE_TEXT,
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"assets": {"photo-1": {"type": "image", "part": "file1"}},
|
||||
},
|
||||
)
|
||||
|
||||
if not deployment_drops_params:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue