diff --git a/litellm/__init__.py b/litellm/__init__.py index 3d6b2638522..bff6588f21c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2294,6 +2294,7 @@ if TYPE_CHECKING: supports_vision: Callable[..., bool] supports_audio_input: Callable[..., bool] supports_video_input: Callable[..., bool] + supports_box_questions: Callable[..., bool] supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e2993100095..d47d72ef7a8 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -38,6 +38,7 @@ UTILS_NAMES: Final = ( "supports_vision", "supports_audio_input", "supports_video_input", + "supports_box_questions", "supports_audio_output", "supports_system_messages", "supports_reasoning", @@ -488,6 +489,7 @@ _UTILS_IMPORT_MAP: Final = { "supports_vision": (".utils", "supports_vision"), "supports_audio_input": (".utils", "supports_audio_input"), "supports_video_input": (".utils", "supports_video_input"), + "supports_box_questions": (".utils", "supports_box_questions"), "supports_audio_output": (".utils", "supports_audio_output"), "supports_system_messages": (".utils", "supports_system_messages"), "supports_reasoning": (".utils", "supports_reasoning"), diff --git a/litellm/constants.py b/litellm/constants.py index 3e855d0bfbf..67bc6bc43fd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1686,6 +1686,7 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = { # e.g., 'x-pass-anthropic-beta: value' becomes 'anthropic-beta: value' # Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.) PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-" +TYPESAFE_DANGER_ZONE_HEADER_PREFIX: Final = "typesafe-danger-zone-" INTERNAL_KWARG_PREFIX: Final = "_litellm_" CONTROL_OPTIONS_KEY: Final = f"{INTERNAL_KWARG_PREFIX}control" diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 0fc703c2c68..0c54422eb68 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final, Literal, TypeAlias import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -10,6 +10,7 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_urls_to_base64, @@ -31,10 +32,17 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir from litellm.types.decisions import ( DecisionQuestion, + DecisionsAsset, + DecisionsAssetReference, + DecisionsAssets, DecisionsAudio, DecisionsBase64Image, DecisionsImage, + DecisionsIRAnswer, + DecisionsIRBoxQuestion, DecisionsIRMessages, + DecisionsIRQuestion, + DecisionsIRRefusal, DecisionsIRRequest, DecisionsIRResponse, DecisionsIRState, @@ -48,12 +56,13 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, UnsupportedDecisionsRequest, ) -from litellm.types.utils import LlmProviders +from litellm.types.utils import LlmProviders, ProviderSpecificHeader from litellm.utils import ProviderConfigManager SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] DecisionsImages: TypeAlias = Sequence[str | DecisionsBase64Image | Mapping[str, object]] +DecisionsAssetEntries: TypeAlias = Mapping[str, DecisionsAsset | Mapping[str, object]] DecisionsAudioClips: TypeAlias = Sequence[DecisionsAudio | Mapping[str, object]] DecisionsVideoClips: TypeAlias = Sequence[DecisionsVideo | Mapping[str, object]] DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody @@ -66,12 +75,22 @@ _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( _IMAGES_ADAPTER: Final[TypeAdapter[tuple[DecisionsImage, ...] | None]] = TypeAdapter( tuple[DecisionsImage, ...] | None, config=ConfigDict(title="images") ) +_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ + TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None] +] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None) _HANDLER: Final = BaseLLMHTTPHandler() _BASE64_MARKER: Final = ";base64," _IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images")) _IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_ASSETS_ADAPTER: Final[TypeAdapter[DecisionsAssets | None]] = TypeAdapter( + DecisionsAssets | None, config=ConfigDict(title="assets") +) +_ASSET_ENTRY_ADAPTER: Final[TypeAdapter[tuple[DecisionsAssetReference, DecisionsAsset]]] = TypeAdapter( + tuple[DecisionsAssetReference, DecisionsAsset] +) +_ASSET_ENTRIES_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) def _valid_image(image: object) -> DecisionsImage | None: @@ -96,6 +115,29 @@ def _no_image(_image: DecisionsImage) -> bool: return False +def _valid_asset(entry: tuple[str, object]) -> tuple[str, DecisionsAsset] | None: + try: + return _ASSET_ENTRY_ADAPTER.validate_python(entry) + except ValidationError: + return None + + +def _asset_entries(assets: object) -> Mapping[str, object]: + try: + return _ASSET_ENTRIES_ADAPTER.validate_python(assets) + except ValidationError: + return MappingProxyType({}) + + +def _request_assets(assets: object, drops_params: bool) -> Mapping[str, DecisionsAsset] | None: + if not drops_params: + return _ASSETS_ADAPTER.validate_python(assets) + valid: Final = MappingProxyType( + dict(entry for entry in map(_valid_asset, _asset_entries(assets).items()) if entry is not None) + ) + return valid or None + + def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]: if not drops_params: return _IMAGES_ADAPTER.validate_python(images) or () @@ -107,6 +149,7 @@ class SystemOneRequestFields: state: DecisionsJSON | None questions: SystemOneQuestions | None images: object = None + assets: object = None audio: DecisionsAudioClips | None = None videos: DecisionsVideoClips | None = None @@ -116,6 +159,7 @@ class SystemOneRequestFields: "state": self.state, "questions": self.questions, "images": _request_images(self.images, drops_params) or None, + "assets": _request_assets(self.assets, drops_params), "audio": self.audio, "videos": self.videos, } @@ -158,6 +202,7 @@ class DecisionsCall: headers: Mapping[str, str] timeout: float | httpx.Timeout | None litellm_params: Mapping[str, object] = field(repr=False) + caller_questions: tuple[DecisionsIRQuestion, ...] def supported_decisions_providers() -> tuple[str, ...]: @@ -229,8 +274,9 @@ class _UnsupportedParams: @dataclass(frozen=True, slots=True) -class _NoInputLeftAfterDrop: +class _NothingLeftAfterDrop: names: tuple[str, ...] + missing: Literal["input to decide on", "questions to answer"] @dataclass(frozen=True, slots=True) @@ -239,12 +285,22 @@ class _MalformedImages: def _unsupported_params( - ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str + ir_request: DecisionsIRRequest, + provider_config: BaseDecisionsConfig, + model: str, + custom_llm_provider: str, + headers: Mapping[str, str], ) -> tuple[str, ...]: refuses_safety_identifier: Final = ( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) - refused_image_param: Final = None if provider_config.supports_images else image_param(ir_request.input) + refused_image_param: Final = ( + None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + ) + refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) + refuses_box_questions: Final = _has_box_question(ir_request.questions) and not ( + provider_config.supports_box_questions(model, custom_llm_provider, headers) + ) refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input( model, custom_llm_provider ) @@ -254,18 +310,42 @@ def _unsupported_params( return ( *(("safety_identifier",) if refuses_safety_identifier else ()), *(() if refused_image_param is None else (refused_image_param,)), + *(("assets",) if refuses_assets else ()), + *(("box questions",) if refuses_box_questions else ()), *(("audio",) if refuses_audio else ()), *(("videos",) if refuses_videos else ()), ) +def _has_box_question(questions: Sequence[DecisionsIRQuestion]) -> bool: + return any(isinstance(question, DecisionsIRBoxQuestion) for question in questions) + + +def _is_not_box_question(question: DecisionsIRQuestion) -> bool: + return not isinstance(question, DecisionsIRBoxQuestion) + + def _without_unsupported_params( - ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig + ir_request: DecisionsIRRequest, + provider_config: BaseDecisionsConfig, + model: str, + custom_llm_provider: str, + headers: Mapping[str, str], ) -> DecisionsIRRequest: return replace( ir_request, - input=ir_request.input if provider_config.supports_images else with_images_where(ir_request.input, _no_image), + input=( + ir_request.input + if provider_config.supports_images_in(ir_request.input, headers) + else with_images_where(ir_request.input, _no_image) + ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, + assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}), + questions=( + ir_request.questions + if provider_config.supports_box_questions(model, custom_llm_provider, headers) + else tuple(filter(_is_not_box_question, ir_request.questions)) + ), ) @@ -286,28 +366,46 @@ def _without_malformed_images( def _provider_ir_request( - request: DecisionsRequestFormat, + ir_request: DecisionsIRRequest, *, - safety_identifier: str | None, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, + headers: Mapping[str, str], kwargs: Mapping[str, object], -) -> DecisionsIRRequest | _UnsupportedParams | _NoInputLeftAfterDrop | _MalformedImages: - ir_request: Final = _without_malformed_images(_ir_request(request, safety_identifier), _drops_params(kwargs)) - if isinstance(ir_request, _MalformedImages): - return ir_request - unsupported: Final = _unsupported_params(ir_request, provider_config, model, custom_llm_provider) +) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop: + unsupported: Final = _unsupported_params(ir_request, provider_config, model, custom_llm_provider, headers) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config) + supported: Final = _without_unsupported_params(ir_request, provider_config, model, custom_llm_provider, headers) if _leaves_no_input(supported.input): - return _NoInputLeftAfterDrop(names=unsupported) + return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") + if not supported.questions: + return _NothingLeftAfterDrop(names=unsupported, missing="questions to answer") return supported +def _provider_specific_header( + value: object, +) -> ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None: + try: + return _PROVIDER_SPECIFIC_HEADER_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _outbound_headers( + extra_headers: Mapping[str, str] | None, custom_llm_provider: str, kwargs: Mapping[str, object] +) -> Mapping[str, str]: + forwarded: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_provider_specific_header(kwargs.get("provider_specific_header")), + custom_llm_provider=custom_llm_provider, + ) + return MappingProxyType({**(extra_headers or {}), **forwarded}) + + def prepare_call( *, model: str, @@ -338,6 +436,7 @@ def prepare_call( except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: + headers: Final = _outbound_headers(extra_headers, provider, kwargs) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) request: Final = _fields_without_unsupported_media( fields, provider_config, canonical_model, provider, kwargs @@ -368,12 +467,24 @@ def prepare_call( llm_provider=provider, ) + caller_ir_request: Final = _without_malformed_images( + _ir_request(request, request_safety_identifier), _drops_params(kwargs) + ) + if isinstance(caller_ir_request, _MalformedImages): + raise litellm.BadRequestError( + message=( + f"Invalid Decisions request: {caller_ir_request.count} input_image part(s) have an image_url that is " + "neither a data:image/...;base64, URL nor an http(s) URL" + ), + model=model, + llm_provider=provider, + ) ir_request: Final = _provider_ir_request( - request, - safety_identifier=request_safety_identifier, + caller_ir_request, provider_config=provider_config, model=canonical_model, custom_llm_provider=provider, + headers=headers, kwargs=kwargs, ) if isinstance(ir_request, _UnsupportedParams): @@ -386,20 +497,11 @@ def prepare_call( model=model, llm_provider=provider, ) - if isinstance(ir_request, _MalformedImages): - raise litellm.BadRequestError( - message=( - f"Invalid Decisions request: {ir_request.count} input_image part(s) have an image_url that is " - "neither a data:image/...;base64, URL nor an http(s) URL" - ), - model=model, - llm_provider=provider, - ) - if isinstance(ir_request, _NoInputLeftAfterDrop): + if isinstance(ir_request, _NothingLeftAfterDrop): raise litellm.BadRequestError( message=( f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, " - "and dropping them leaves no input to decide on" + f"and dropping them leaves no {ir_request.missing}" ), model=model, llm_provider=provider, @@ -425,9 +527,10 @@ def prepare_call( api_base=resolved_api_base, api_key=resolved_api_key, logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, - headers=extra_headers or {}, + headers=headers, timeout=timeout, litellm_params=kwargs, + caller_questions=caller_ir_request.questions, ) @@ -561,14 +664,24 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci return formatted +def _caller_answers(call: DecisionsCall, answers: Sequence[DecisionsIRAnswer]) -> tuple[DecisionsIRAnswer, ...]: + sent_answers: Final = iter(answers) + return tuple( + next(sent_answers) if question in call.ir_request.questions else DecisionsIRRefusal() + for question in call.caller_questions + ) + + def _formatted_response( response: DecisionsIRResponse, call: DecisionsCall ) -> DecisionsResponse | OpenAIDecisionResponse: + caller_request: Final = replace(call.ir_request, questions=call.caller_questions) + caller_response: Final = replace(response, answers=_caller_answers(call, response.answers)) match call.request: case DecisionsRequestBody(): - return ir_to_systemone_response(response, call.ir_request) + return ir_to_systemone_response(caller_response, caller_request) case OpenAIDecisionRequestBody(): - return ir_to_openai_response(response, call.ir_request, call.requested_model) + return ir_to_openai_response(caller_response, caller_request, call.requested_model) case _: assert_never(call.request) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 6b278f03030..553fe4b884a 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -27,7 +27,7 @@ _SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = Ty _OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( OpenAIDecisionQuestions | None ) -_SYSTEMONE_FIELDS: Final = frozenset({"state", "images"}) +_SYSTEMONE_FIELDS: Final = frozenset({"state", "images", "assets"}) def _request_fields( @@ -55,6 +55,7 @@ def _request_fields( state=_STATE_ADAPTER.validate_python(kwargs["state"]), questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions), images=kwargs.get("images"), + assets=kwargs.get("assets"), audio=audio, videos=videos, ) diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index 2618aee9afa..20afd7535f3 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -1,4 +1,4 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final from litellm.types.utils import ProviderSpecificHeader @@ -9,7 +9,7 @@ class ProviderSpecificHeaderUtils: def get_provider_specific_headers( provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, custom_llm_provider: str | None, - ) -> dict: + ) -> Mapping[str, str]: """ Get the provider specific headers for the given custom llm provider. diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 564bcc18275..f0abe56b88d 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -14,6 +14,8 @@ import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( + BoxAnswer, + BoxQuestion, ChoiceAnswer, ChoiceQuestion, DecisionAnswer, @@ -21,6 +23,8 @@ from litellm.types.decisions import ( DecisionsBase64Image, DecisionsImage, DecisionsIRAnswer, + DecisionsIRBoxAnswer, + DecisionsIRBoxQuestion, DecisionsIRChoiceAnswer, DecisionsIRChoiceOption, DecisionsIRChoiceProbability, @@ -98,6 +102,8 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: ), extra=extra, ) + case BoxQuestion(): + return DecisionsIRBoxQuestion(name=name, instructions=question.instructions, extra=extra) case _: assert_never(question) @@ -105,6 +111,7 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest: return DecisionsIRRequest( input=DecisionsIRState(state=request.state, images=tuple(request.images or ())), + assets=MappingProxyType(dict(request.assets or {})), audio=request.audio, videos=request.videos, questions=tuple(_ir_question(name, question) for name, question in request.questions.items()), @@ -282,6 +289,8 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: "criteria": [_level_criterion(level) for level in question.levels], **question.extra, } + case DecisionsIRBoxQuestion(): + return {"type": "box", "instructions": question.instructions, **question.extra} case _: assert_never(question) @@ -311,6 +320,11 @@ def ir_to_systemone_request( ), "questions": {key: _systemone_question(question) for key, question in keyed_questions}, **({"images": [_systemone_image(image) for image in images]} if images else {}), + **( + {"assets": {key: asset.model_dump(exclude_none=True) for key, asset in request.assets.items()}} + if request.assets + else {} + ), } @@ -351,6 +365,10 @@ def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> return _ir_choice_answer(question, answer) case DecisionsIRScoreQuestion(), ScoreAnswer(): return _ir_score_answer(question, answer) + case DecisionsIRBoxQuestion(), BoxAnswer(): + return DecisionsIRBoxAnswer( + box=answer.box, confidence=answer.confidence, extra=MappingProxyType(answer.model_extra or {}) + ) case _: return DecisionsIRRefusal() @@ -403,6 +421,10 @@ def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None: "probabilities": {str(item.value): item.probability for item in answer.probabilities}, } ) + case DecisionsIRBoxAnswer(): + return BoxAnswer.model_validate( + {**answer.extra, "type": "box", "box": answer.box, "confidence": answer.confidence} + ) case DecisionsIRRefusal(): return None case _: @@ -507,6 +529,19 @@ class BaseDecisionsConfig(ABC): ) -> tuple[Mapping[str, str], bytes | None]: return headers, None + def supports_images_in( + self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + ) -> bool: + return self.supports_images + + def supports_assets(self, headers: Mapping[str, str]) -> bool: + return False + + def supports_box_questions(self, model: str, custom_llm_provider: str, headers: Mapping[str, str]) -> bool: + return _is_decisions_model(model, custom_llm_provider) and litellm.supports_box_questions( + model=model, custom_llm_provider=custom_llm_provider + ) + def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input( model=model, custom_llm_provider=custom_llm_provider diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index a0ffd7f0924..95511f5221b 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -10,6 +10,8 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionsImage, DecisionsIRAnswer, + DecisionsIRBoxAnswer, + DecisionsIRBoxQuestion, DecisionsIRChoiceAnswer, DecisionsIRChoiceOption, DecisionsIRChoiceProbability, @@ -28,6 +30,8 @@ from litellm.types.decisions import ( DecisionsIRState, DecisionsIRUsage, DecisionsJSON, + OpenAIBoxAnswer, + OpenAIBoxQuestion, OpenAIChoiceAnswer, OpenAIChoiceProbability, OpenAIChoiceQuestion, @@ -85,6 +89,8 @@ def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion: DecisionsIRScoreLevel(label=level.label, description=level.description) for level in question.levels ), ) + case OpenAIBoxQuestion(): + return DecisionsIRBoxQuestion(name=question.name, instructions=question.instructions) case _: assert_never(question) @@ -145,6 +151,12 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: for level in question.levels ], } + case DecisionsIRBoxQuestion(): + return { + "type": "box", + **_text_field("name", question.name), + "instructions": decisions_text(question.instructions), + } case _: assert_never(question) @@ -287,6 +299,8 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> return _openai_choice_answer(question, answer) case DecisionsIRScoreQuestion(), DecisionsIRScoreAnswer(): return _openai_score_answer(question, answer) + case DecisionsIRBoxQuestion(), DecisionsIRBoxAnswer(): + return OpenAIBoxAnswer(name=question.name, box=answer.box, confidence=answer.confidence) case _: return OpenAIRefusalAnswer(name=question.name) diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index e71c526a840..63e684db422 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -1,9 +1,106 @@ -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +import hashlib +from collections.abc import Mapping +from dataclasses import replace +from itertools import chain +from types import MappingProxyType +from typing import Final, Literal + +from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type +from litellm.types.decisions import ( + DecisionsAsset, + DecisionsIRMessages, + DecisionsIRRequest, + DecisionsIRState, + OpenAIDecisionInputImage, + OpenAIDecisionInputMessage, + OpenAIDecisionInputText, + UnsupportedDecisionsRequest, +) + +_IMAGES_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}images" +_ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType( + {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"} +) +_URL_SAFE_TO_STANDARD_BASE64: Final = str.maketrans("-_", "+/") + + +def _image_reference(image_url: str) -> str: + return f"image-{hashlib.sha256(image_url.encode()).hexdigest()[:32]}" + + +def _image_asset(image_url: str) -> DecisionsAsset | None: + asset_format: Final = _ASSET_FORMATS.get(image_media_type(image_url) or "") + if asset_format is None: + return None + return DecisionsAsset( + type="image", + format=asset_format, + data_base64=image_url.partition(",")[2].translate(_URL_SAFE_TO_STANDARD_BASE64), + ) + + +def _part_text(part: OpenAIDecisionInputText | OpenAIDecisionInputImage) -> str: + if isinstance(part, OpenAIDecisionInputText): + return part.text + return part.image_url if _image_asset(part.image_url) is None else _image_reference(part.image_url) + + +def _message_texts(message: OpenAIDecisionInputMessage) -> tuple[str, ...]: + if isinstance(message.content, str): + return (message.content,) + return tuple(map(_part_text, message.content)) + + +def _message_assets(message: OpenAIDecisionInputMessage) -> Mapping[str, DecisionsAsset]: + if isinstance(message.content, str): + return {} + return { + _image_reference(part.image_url): asset + for part in message.content + if isinstance(part, OpenAIDecisionInputImage) and (asset := _image_asset(part.image_url)) is not None + } + + +def _with_images_as_assets(request: DecisionsIRRequest) -> DecisionsIRRequest: + if not isinstance(request.input, DecisionsIRMessages): + return request + messages: Final = request.input.messages + return replace( + request, + input=DecisionsIRState(state="\n\n".join(chain.from_iterable(map(_message_texts, messages)))), + assets=MappingProxyType( + dict(chain(request.assets.items(), *(_message_assets(message).items() for message in messages))) + ), + ) class TypeSafeDecisionsConfig(BaseDecisionsConfig): api_key_env = ("TYPESAFE_API_KEY",) api_base_env = ("TYPESAFE_API_BASE",) + image_media_types = frozenset(_ASSET_FORMATS) + inlines_remote_images = True def get_default_api_base(self) -> str | None: return "https://api.typesafe.ai" + + def supports_images_in( + self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + ) -> bool: + return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers) + + def supports_assets(self, headers: Mapping[str, str]) -> bool: + return any( + name.lower() == _IMAGES_HEADER and value.strip().lower() == "true" for name, value in headers.items() + ) + + def supports_box_questions(self, model: str, custom_llm_provider: str, headers: Mapping[str, str]) -> bool: + return self.supports_assets(headers) and super().supports_box_questions(model, custom_llm_provider, headers) + + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return super().transform_decisions_request(model, _with_images_as_assets(request), custom_llm_provider) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5621b51768b..f4fffa665f0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -69875,7 +69875,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-latest": { "input_cost_per_token": 4.2e-08, @@ -69891,7 +69892,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c9422e070c8..ce3c721c014 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9649,6 +9649,37 @@ "decisions": { "components": { "schemas": { + "BoxAnswer": { + "additionalProperties": true, + "properties": { + "box": { + "$ref": "#/components/schemas/DecisionsBox" + }, + "confidence": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" + }, + "type": { + "const": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "box", + "confidence" + ], + "title": "BoxAnswer", + "type": "object" + }, "ChoiceAnswer": { "additionalProperties": true, "properties": { @@ -9682,6 +9713,35 @@ "title": "ChoiceAnswer", "type": "object" }, + "DecisionsBox": { + "additionalProperties": true, + "properties": { + "bottom": { + "title": "Bottom", + "type": "number" + }, + "left": { + "title": "Left", + "type": "number" + }, + "right": { + "title": "Right", + "type": "number" + }, + "top": { + "title": "Top", + "type": "number" + } + }, + "required": [ + "top", + "bottom", + "left", + "right" + ], + "title": "DecisionsBox", + "type": "object" + }, "DecisionsResponse": { "additionalProperties": true, "properties": { @@ -9689,6 +9749,7 @@ "additionalProperties": { "discriminator": { "mapping": { + "box": "#/components/schemas/BoxAnswer", "choice": "#/components/schemas/ChoiceAnswer", "noul": "#/components/schemas/NoulAnswer", "score": "#/components/schemas/ScoreAnswer" @@ -9704,6 +9765,9 @@ }, { "$ref": "#/components/schemas/ScoreAnswer" + }, + { + "$ref": "#/components/schemas/BoxAnswer" } ] }, @@ -9775,6 +9839,48 @@ "title": "NoulAnswer", "type": "object" }, + "OpenAIBoxAnswer": { + "properties": { + "box": { + "$ref": "#/components/schemas/DecisionsBox" + }, + "confidence": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "box", + "default": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "box", + "confidence" + ], + "title": "OpenAIBoxAnswer", + "type": "object" + }, "OpenAIChoiceAnswer": { "properties": { "choice": { @@ -9885,6 +9991,7 @@ "items": { "discriminator": { "mapping": { + "box": "#/components/schemas/OpenAIBoxAnswer", "choice": "#/components/schemas/OpenAIChoiceAnswer", "predicate": "#/components/schemas/OpenAIPredicateAnswer", "refusal": "#/components/schemas/OpenAIRefusalAnswer", @@ -9902,6 +10009,9 @@ { "$ref": "#/components/schemas/OpenAIScoreAnswer" }, + { + "$ref": "#/components/schemas/OpenAIBoxAnswer" + }, { "$ref": "#/components/schemas/OpenAIRefusalAnswer" } @@ -10144,7 +10254,7 @@ "paths": { "/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", "requestBody": { "content": { @@ -10518,6 +10628,39 @@ ], "title": "OpenAIScoreQuestion", "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "What to find in the input's image", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIBoxQuestion", + "type": "object" } ] }, @@ -10614,7 +10757,7 @@ }, "/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", "requestBody": { "content": { @@ -10649,6 +10792,59 @@ "schema": { "additionalProperties": true, "properties": { + "assets": { + "anyOf": [ + { + "patternProperties": { + "\\S": { + "additionalProperties": false, + "properties": { + "data_base64": { + "description": "Base64-encoded image bytes, no data: prefix", + "pattern": "^[A-Za-z0-9+/]+={0,2}$", + "title": "Data Base64", + "type": "string" + }, + "format": { + "description": "The image's format, in lowercase", + "enum": [ + "png", + "jpg", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "anyOf": [ + { + "const": "image", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "required": [ + "format", + "data_base64" + ], + "title": "DecisionsAsset", + "type": "object" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "TypeSafe only. Images keyed by a reference string. Write the reference in state, instructions or criteria where the image belongs", + "title": "Assets" + }, "audio": { "default": [], "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", @@ -10693,7 +10889,7 @@ "items": { "anyOf": [ { - "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$", + "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,[A-Za-z0-9+/_-]+={0,2}|https?://\\S+)$", "type": "string" }, { @@ -10930,6 +11126,39 @@ ], "title": "ScoreQuestion", "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "What to find, and in which image, such as 'the coffee mug in `photo`'", + "title": "Instructions" + }, + "type": { + "const": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "BoxQuestion", + "type": "object" } ] }, @@ -11035,7 +11264,7 @@ }, "/v1/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", "requestBody": { "content": { @@ -11409,6 +11638,39 @@ ], "title": "OpenAIScoreQuestion", "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "What to find in the input's image", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIBoxQuestion", + "type": "object" } ] }, @@ -11505,7 +11767,7 @@ }, "/v1/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", "requestBody": { "content": { @@ -11540,6 +11802,59 @@ "schema": { "additionalProperties": true, "properties": { + "assets": { + "anyOf": [ + { + "patternProperties": { + "\\S": { + "additionalProperties": false, + "properties": { + "data_base64": { + "description": "Base64-encoded image bytes, no data: prefix", + "pattern": "^[A-Za-z0-9+/]+={0,2}$", + "title": "Data Base64", + "type": "string" + }, + "format": { + "description": "The image's format, in lowercase", + "enum": [ + "png", + "jpg", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "anyOf": [ + { + "const": "image", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "required": [ + "format", + "data_base64" + ], + "title": "DecisionsAsset", + "type": "object" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "TypeSafe only. Images keyed by a reference string. Write the reference in state, instructions or criteria where the image belongs", + "title": "Assets" + }, "audio": { "default": [], "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", @@ -11584,7 +11899,7 @@ "items": { "anyOf": [ { - "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,.+|https?://\\S+)$", + "pattern": "^(?i:data:image/[^;,\\s]+(?:;[^;,\\s]+)*;base64,[A-Za-z0-9+/_-]+={0,2}|https?://\\S+)$", "type": "string" }, { @@ -11821,6 +12136,39 @@ ], "title": "ScoreQuestion", "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "What to find, and in which image, such as 'the coffee mug in `photo`'", + "title": "Instructions" + }, + "type": { + "const": "box", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "BoxQuestion", + "type": "object" } ] }, diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 5eb070cb789..0ff77fc9a30 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -92,7 +92,7 @@ async def _invalid_request( ) -_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "audio", "videos"}) +_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "assets", "audio", "videos"}) def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: @@ -200,9 +200,10 @@ async def systemone( Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with - `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, - `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider - is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`). + TypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break + these rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. [Docs](https://docs.litellm.ai/docs/decisions) @@ -243,8 +244,9 @@ async def decisions( Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with - `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these - rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the + `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before + any provider is called. `model` is any decision model in the proxy model list. System One decision models accept this format too, LiteLLM translates the request and the answers. Streaming is not supported. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8633eca028f..bfda00aa39f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -38,6 +38,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + TYPESAFE_DANGER_ZONE_HEADER_PREFIX, X_LITELLM_DISABLE_CALLBACKS, ) from litellm.integrations.custom_guardrail import without_server_streaming_classification @@ -4025,7 +4026,7 @@ _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_F def add_provider_specific_headers_to_request( data: dict, - headers: dict, + headers: Mapping[str, str], ) -> bool: from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key @@ -4035,12 +4036,18 @@ def add_provider_specific_headers_to_request( for header, value in headers.items() if header.lower() == "authorization" and is_anthropic_oauth_key(value) } + typesafe_danger_zone_headers: Final = { + header: value + for header, value in headers.items() + if header.lower().startswith(TYPESAFE_DANGER_ZONE_HEADER_PREFIX) + } scoped_headers: Final = [ ProviderSpecificHeader(custom_llm_provider=providers, extra_headers=extra_headers) for providers, extra_headers in ( (_ANTHROPIC_API_HEADER_PROVIDERS, anthropic_api_headers), (_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS, anthropic_oauth_credential_headers), + (LlmProviders.TYPESAFE.value, typesafe_danger_zone_headers), ) if extra_headers ] diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 70cee344c91..89e0f01076e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -43,6 +43,7 @@ from litellm.constants import ( AZURE_SPEECH_STT_DOMAIN, AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, + TYPESAFE_DANGER_ZONE_HEADER_PREFIX, ) from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix @@ -654,10 +655,16 @@ async def typesafe_proxy_route( custom_llm_provider="typesafe", region_name=None, ) + danger_zone_headers: Final = { + name: value + for name, value in request.headers.items() + if name.lower().startswith(TYPESAFE_DANGER_ZONE_HEADER_PREFIX) + } endpoint_func: Final = create_pass_through_route( endpoint=endpoint, target=str(updated_url), custom_headers={ + **danger_zone_headers, "Authorization": f"Bearer {typesafe_api_key}", "Content-Type": "application/json", }, diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 88a8adc0a1d..c91f504eefa 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -4,6 +4,7 @@ from typing import Final import httpx from litellm.decisions.call import ( + DecisionsAssetEntries, DecisionsAudioClips, DecisionsCall, DecisionsImages, @@ -25,6 +26,7 @@ def _prepare_systemone_call( state: DecisionsJSON | None, questions: SystemOneQuestions | None, images: DecisionsImages | None, + assets: DecisionsAssetEntries | None, safety_identifier: str | None, audio: DecisionsAudioClips | None, videos: DecisionsVideoClips | None, @@ -38,7 +40,9 @@ def _prepare_systemone_call( reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields(state=state, questions=questions, images=images, audio=audio, videos=videos), + fields=SystemOneRequestFields( + state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos + ), safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, @@ -68,6 +72,7 @@ async def asystemone( audio: DecisionsAudioClips | None = None, videos: DecisionsVideoClips | None = None, images: DecisionsImages | None = None, + assets: DecisionsAssetEntries | None = None, **kwargs: object, # kwargs-ok: litellm_params forwarded by @client ) -> DecisionsResponse: call: Final = _prepare_systemone_call( @@ -75,6 +80,7 @@ async def asystemone( state=state, questions=questions, images=images, + assets=assets, safety_identifier=safety_identifier, audio=audio, videos=videos, @@ -102,6 +108,7 @@ def systemone( audio: DecisionsAudioClips | None = None, videos: DecisionsVideoClips | None = None, images: DecisionsImages | None = None, + assets: DecisionsAssetEntries | None = None, **kwargs: object, # kwargs-ok: litellm_params forwarded by @client ) -> DecisionsResponse: call: Final = _prepare_systemone_call( @@ -109,6 +116,7 @@ def systemone( state=state, questions=questions, images=images, + assets=assets, safety_identifier=safety_identifier, audio=audio, videos=videos, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 6dc4ab78ac8..a1d48f730c0 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config +from pydantic import AfterValidator, ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -11,6 +11,7 @@ from litellm.types.llms.base import LiteLLMPydanticObjectBase DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] MAX_DECISION_QUESTIONS: Final = 128 +MAX_ASSET_REFERENCE_BYTES: Final = 1024 class NoulQuestion(LiteLLMPydanticObjectBase): @@ -56,8 +57,17 @@ class ScoreQuestion(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) +class BoxQuestion(LiteLLMPydanticObjectBase): + type: Literal["box"] + instructions: Annotated[ + DecisionsJSON, Field(description="What to find, and in which image, such as 'the coffee mug in `photo`'") + ] + + model_config = ConfigDict(extra="allow", frozen=True) + + DecisionQuestion: TypeAlias = Annotated[ - NoulQuestion | ChoiceQuestion | ScoreQuestion, + NoulQuestion | ChoiceQuestion | ScoreQuestion | BoxQuestion, Field(discriminator="type"), ] @@ -75,11 +85,33 @@ class DecisionsBase64Image(LiteLLMPydanticObjectBase): DecisionsImage: TypeAlias = ( - Annotated[str, Field(pattern=r"^(?i:data:image/[^;,\s]+(?:;[^;,\s]+)*;base64,.+|https?://\S+)$")] + Annotated[ + str, Field(pattern=r"^(?i:data:image/[^;,\s]+(?:;[^;,\s]+)*;base64,[A-Za-z0-9+/_-]+={0,2}|https?://\S+)$") + ] | DecisionsBase64Image ) +class DecisionsAsset(LiteLLMPydanticObjectBase): + type: Literal["image"] | None = None + format: Annotated[Literal["png", "jpg", "jpeg"], Field(description="The image's format, in lowercase")] + data_base64: Annotated[ + str, Field(pattern=r"^[A-Za-z0-9+/]+={0,2}$", description="Base64-encoded image bytes, no data: prefix") + ] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +def _fits_reference_limit(reference: str) -> str: + if len(reference.encode()) > MAX_ASSET_REFERENCE_BYTES: + raise ValueError(f"an asset reference can be at most {MAX_ASSET_REFERENCE_BYTES} bytes of UTF-8") + return reference + + +DecisionsAssetReference: TypeAlias = Annotated[str, Field(pattern=r"\S"), AfterValidator(_fits_reference_limit)] +DecisionsAssets: TypeAlias = Mapping[DecisionsAssetReference, DecisionsAsset] + + class DecisionsBase64Audio(LiteLLMPydanticObjectBase): content_type: Annotated[str, Field(pattern=r"^audio/\S+$", description="An audio/* media type, such as audio/wav")] base64: Annotated[str, Field(min_length=1, description="Base64-encoded audio bytes")] @@ -135,6 +167,15 @@ class DecisionsRequestBody(DecisionsMediaBody): Sequence[DecisionsImage] | None, Field(description="Images as data URLs or content_type plus base64, for providers that look at images"), ] = None + assets: Annotated[ + DecisionsAssets | None, + Field( + description=( + "TypeSafe only. Images keyed by a reference string. Write the reference in state, instructions " + "or criteria where the image belongs" + ) + ), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -149,6 +190,7 @@ class DecisionsCallParams(TypedDict, total=False): state: Required[ReadOnly[DecisionsJSON]] questions: Required[ReadOnly[DecisionQuestionMap]] images: ReadOnly[Sequence[DecisionsImage] | None] + assets: ReadOnly[DecisionsAssets | None] audio: ReadOnly[Sequence[DecisionsAudio]] videos: ReadOnly[Sequence[DecisionsVideo]] api_key: ReadOnly[str | None] @@ -184,8 +226,25 @@ class ScoreAnswer(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsBox(LiteLLMPydanticObjectBase): + top: float + bottom: float + left: float + right: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class BoxAnswer(LiteLLMPydanticObjectBase): + type: Literal["box"] + box: DecisionsBox + confidence: float | None + + model_config = ConfigDict(extra="allow", frozen=True) + + DecisionAnswer: TypeAlias = Annotated[ - NoulAnswer | ChoiceAnswer | ScoreAnswer, + NoulAnswer | ChoiceAnswer | ScoreAnswer | BoxAnswer, Field(discriminator="type"), ] @@ -311,8 +370,16 @@ class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="forbid", frozen=True) +class OpenAIBoxQuestion(LiteLLMPydanticObjectBase): + type: Literal["box"] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="What to find in the input's image")] + + model_config = ConfigDict(extra="forbid", frozen=True) + + OpenAIDecisionQuestion: TypeAlias = Annotated[ - OpenAIPredicateQuestion | OpenAIChoiceQuestion | OpenAIScoreQuestion, + OpenAIPredicateQuestion | OpenAIChoiceQuestion | OpenAIScoreQuestion | OpenAIBoxQuestion, Field(discriminator="type"), ] @@ -394,8 +461,17 @@ class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) +class OpenAIBoxAnswer(LiteLLMPydanticObjectBase): + type: Literal["box"] = "box" + name: str | None + box: DecisionsBox + confidence: float | None + + model_config = ConfigDict(frozen=True) + + OpenAIDecisionAnswer: TypeAlias = Annotated[ - OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer, + OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIBoxAnswer | OpenAIRefusalAnswer, Field(discriminator="type"), ] @@ -432,6 +508,7 @@ class OpenAIDecisionResponse(_HiddenParamsResponse): _NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) +_NO_ASSETS: Final[Mapping[str, DecisionsAsset]] = MappingProxyType({}) @dataclass(frozen=True, slots=True) @@ -481,7 +558,16 @@ class DecisionsIRScoreQuestion: extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) -DecisionsIRQuestion: TypeAlias = DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion +@dataclass(frozen=True, slots=True) +class DecisionsIRBoxQuestion: + name: str | None + instructions: DecisionsJSON + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +DecisionsIRQuestion: TypeAlias = ( + DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion | DecisionsIRBoxQuestion +) @dataclass(frozen=True, slots=True) @@ -489,6 +575,7 @@ class DecisionsIRRequest: input: DecisionsIRState | DecisionsIRMessages questions: tuple[DecisionsIRQuestion, ...] safety_identifier: str | None = None + assets: Mapping[str, DecisionsAsset] = field(default_factory=lambda: _NO_ASSETS) audio: tuple[DecisionsAudio, ...] = () videos: tuple[DecisionsVideo, ...] = () @@ -528,13 +615,24 @@ class DecisionsIRScoreAnswer: extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) +@dataclass(frozen=True, slots=True) +class DecisionsIRBoxAnswer: + box: DecisionsBox + confidence: float | None + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + @dataclass(frozen=True, slots=True) class DecisionsIRRefusal: pass DecisionsIRAnswer: TypeAlias = ( - DecisionsIRPredicateAnswer | DecisionsIRChoiceAnswer | DecisionsIRScoreAnswer | DecisionsIRRefusal + DecisionsIRPredicateAnswer + | DecisionsIRChoiceAnswer + | DecisionsIRScoreAnswer + | DecisionsIRBoxAnswer + | DecisionsIRRefusal ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d5b8a7c7db9..97796153cf2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -196,6 +196,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_computer_use: bool | None supports_audio_input: bool | None supports_video_input: ReadOnly[bool | None] + supports_box_questions: ReadOnly[bool | None] supports_embedding_image_input: bool | None supports_audio_output: bool | None supports_pdf_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 596fd96b021..0b9b0d79138 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3015,6 +3015,10 @@ def supports_video_input(model: str, custom_llm_provider: str | None = None) -> return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_video_input") +def supports_box_questions(model: str, custom_llm_provider: str | None = None) -> bool: + return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_box_questions") + + def supports_pdf_input(model: str, custom_llm_provider: str | None = None) -> bool: """Check if a given model supports pdf input in a chat completion call""" return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_pdf_input") @@ -6445,6 +6449,7 @@ def get_model_info_helper( supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None), supports_audio_input=_model_info.get("supports_audio_input", None), supports_video_input=_model_info.get("supports_video_input", None), + supports_box_questions=_model_info.get("supports_box_questions", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), @@ -6596,6 +6601,7 @@ def get_model_info( supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] supports_video_input: Optional[bool] + supports_box_questions: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] supports_web_search: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5621b51768b..f4fffa665f0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -69875,7 +69875,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-latest": { "input_cost_per_token": 4.2e-08, @@ -69891,7 +69892,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 9f5db5abcda..6437884e0d0 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1032,6 +1032,9 @@ "supports_bedrock_runtime_chat_completions_tools_with_reasoning": { "type": "boolean" }, + "supports_box_questions": { + "type": "boolean" + }, "supports_computer_use": { "type": "boolean" }, diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 955f11f4368..8e39523d934 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -16,6 +16,7 @@ import respx from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials +from respx.models import Call import litellm from litellm.constants import AWS_SIGNING_MAX_THREADS @@ -3071,3 +3072,133 @@ async def test_fireworks_ai_bills_cached_input_tokens_at_the_cache_read_rate( assert recording_logger.standard_logging_object is not None assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(uncached + cached) assert recording_logger.standard_logging_object["response_cost"] < 300 * float(prices["input_cost_per_token"]) + + +_BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + ( + "decisions", + MappingProxyType( + { + "input": "review", + "questions": ( + *_PREDICATE_QUESTIONS, + {"type": "box", "name": "crack", "instructions": "the crack on the screen"}, + ), + } + ), + ), + ( + "systemone", + MappingProxyType( + { + "state": "review", + "questions": { + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "crack": {"type": "box", "instructions": "the crack on the screen"}, + }, + } + ), + ), +) +_UNSUPPORTED_JEV_PARAMS: Final[tuple[tuple[str, Mapping[str, object], str], ...]] = ( + *((api, request_kwargs, "box questions") for api, request_kwargs in _BOX_QUESTION_REQUESTS), + ( + "systemone", + MappingProxyType( + { + "state": "review photo-1", + "questions": _SYSTEMONE_PREDICATE, + "assets": {"photo-1": {"format": "png", "data_base64": "AA=="}}, + } + ), + "assets", + ), +) + + +@pytest.mark.parametrize( + ("api", "request_kwargs", "param"), + _UNSUPPORTED_JEV_PARAMS, + ids=("decisions_box", "systemone_box", "systemone_assets"), +) +def test_assets_and_box_questions_are_refused_before_http_when_the_provider_cannot_take_them( + api: str, + request_kwargs: Mapping[str, object], + param: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: + getattr(litellm, api)(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +async def test_assets_are_dropped_from_the_wire_under_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + await litellm.asystemone( + model="perplexity/pplx-decider-v1-27b", + state="review photo-1", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + assets={"photo-1": {"format": "png", "data_base64": "AA=="}}, + api_key="caller-key", + drop_params=True, + ) + + sent: Final[Call] = route.calls.last + assert json.loads(sent.request.content) == { + "model": "pplx-decider-v1-27b", + "state": "review photo-1", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +_DROPPED_BOX_ANSWERS: Final[Mapping[str, object]] = MappingProxyType( + { + "decisions": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "crack"}, + ], + "systemone": {"is_defect": {"type": "noul", "noul": 0.9}}, + } +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("api", "request_kwargs"), _BOX_QUESTION_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_box_questions_are_dropped_under_drop_params_and_answered_as_refusals( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + response: Final = await getattr(litellm, f"a{api}")( + model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs + ) + + sent: Final[Call] = route.calls.last + assert json.loads(sent.request.content)["questions"] == { + "is_defect": {"type": "noul", "instructions": "Is this a defect?"} + } + assert response.model_dump(mode="json")["answers"] == _DROPPED_BOX_ANSWERS[api] + + +def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="leaves no questions to answer") as caught: + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + input="review", + questions=[{"type": "box", "name": "crack", "instructions": "the crack on the screen"}], + api_key="caller-key", + drop_params=True, + ) + + assert caught.value.status_code == 400 + assert not route.called diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index a729ac41371..d0219ea2963 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from typing import Final import pytest -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm.llms.base_llm.decisions.transformation import ( ir_to_systemone_request, @@ -195,7 +195,9 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: audio: Final = ["data:audio/wav;base64,YQ=="] videos: Final = ["data:video/mp4;base64,YQ=="] audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio})) - video_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})) + video_only: Final = systemone_request_to_ir( + _SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos}) + ) assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == { "model": "clef-omni", @@ -207,5 +209,58 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: **_SYSTEMONE_REQUEST, "videos": videos, } - assert isinstance(ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest) - assert isinstance(ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest) + assert isinstance( + ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest + ) + assert isinstance( + ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest + ) + + +_ASSET_REQUEST: Final[Mapping[str, object]] = { + "state": {"photo": "img-7f3a"}, + "questions": {"has_square": {"type": "noul", "instructions": "Is there a red square in `photo`?"}}, +} + + +@pytest.mark.parametrize( + "assets", + [ + {" ": {"type": "image", "format": "png", "data_base64": "iVBORw0KGgo="}}, + {"r" * 1025: {"type": "image", "format": "png", "data_base64": "iVBORw0KGgo="}}, + {"img-7f3a": {"type": "image", "format": "png", "data_base64": "not base64!"}}, + ], + ids=["blank-reference", "reference-over-1024-bytes", "bad-base64"], +) +def test_assets_typesafe_would_reject_are_rejected_by_validation(assets: Mapping[str, object]) -> None: + with pytest.raises(ValidationError): + _SYSTEMONE_BODY.validate_python({**_ASSET_REQUEST, "assets": assets}) + + +def test_an_asset_reference_of_exactly_1024_bytes_is_accepted() -> None: + reference: Final = "r" * 1024 + body: Final = _SYSTEMONE_BODY.validate_python( + {**_ASSET_REQUEST, "assets": {reference: {"type": "image", "format": "png", "data_base64": "iVBO"}}} + ) + + assert tuple(body.assets or {}) == (reference,) + + +def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None: + request: Final = {**_ASSET_REQUEST, "questions": {"square": {"type": "box", "instructions": "the square"}}} + payload: Final = { + "model": "jev-1.14.0", + "answers": { + "square": { + "type": "box", + "box": {"top": 0.5, "bottom": 0.75, "left": 0.25, "right": 0.5}, + "confidence": None, + } + }, + "usage": {"input_tokens": 737, "output_tokens": 20}, + } + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(request)) + + response: Final = ir_to_systemone_response(parse_systemone_response(payload, ir), ir) + + assert response.model_dump(mode="json") == payload diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 8414b6c0e02..bd5e24d861c 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1001,6 +1001,38 @@ def test_malformed_images_are_refused_unless_the_deployment_drops_params( } +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_malformed_assets_for_a_deployment_without_asset_support_are_dropped_only_when_it_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_decider_to( + monkeypatch, + {"model": "perplexity/pplx-decider-v1-27b", "api_key": "test-key", "drop_params": deployment_drops_params}, + ) + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/systemone", + json={**_IMAGE_REQUEST_BODIES[1], "assets": {"photo-1": {"type": "image", "part": "file1"}}}, + ) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "assets" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": _IMAGE_TEXT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + @pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) @pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) def test_clips_for_a_model_without_media_support_are_refused_unless_the_deployment_drops_params( diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index e2ecfbb8ba9..9ef5f16a240 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7822,6 +7822,20 @@ class TestTypeSafePassthroughRoute: assert sent.headers["authorization"] == "Bearer typesafe-test-key" assert json.loads(sent.content or b"{}") == (body or {}) + def test_forwards_only_the_typesafe_danger_zone_headers_upstream(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://typesafe.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + client.post( + "/typesafe/v1/systemone", + json={"state": "x"}, + headers={"TypeSafe-Danger-Zone-Images": "true", "X-Unrelated": "1"}, + ) + + sent: Final = route.calls.last.request.headers + assert (sent.get("typesafe-danger-zone-images"), sent.get("x-unrelated")) == ("true", None) + @pytest.mark.parametrize( "provider, endpoint, is_decision_request", ( diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 04c12120952..809ade05613 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -7848,6 +7848,23 @@ def test_client_anthropic_api_headers_stay_off_openai_compatible_providers(): assert forwarded == {} +TYPESAFE_DANGER_ZONE_HEADERS = { + "TypeSafe-Danger-Zone-Images": "true", + "typesafe-danger-zone-long-context": "true", +} + + +def test_client_typesafe_danger_zone_headers_reach_typesafe(): + client_headers = {**TYPESAFE_DANGER_ZONE_HEADERS, "x-request-id": "abc"} + + assert _headers_forwarded_to(client_headers, "typesafe") == TYPESAFE_DANGER_ZONE_HEADERS + + +@pytest.mark.parametrize("custom_llm_provider", ["perplexity", "openai", "anthropic"]) +def test_client_typesafe_danger_zone_headers_stay_off_other_providers(custom_llm_provider: str): + assert _headers_forwarded_to(TYPESAFE_DANGER_ZONE_HEADERS, custom_llm_provider) == {} + + @pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) def test_add_provider_specific_headers_reports_a_forwarded_oauth_credential(authorization_header_name): assert add_provider_specific_headers_to_request(data={}, headers=_client_headers(authorization_header_name)) is True diff --git a/tests/unit/systemone/test_main.py b/tests/unit/systemone/test_main.py index 8b7537308c1..82bb3d666cc 100644 --- a/tests/unit/systemone/test_main.py +++ b/tests/unit/systemone/test_main.py @@ -14,6 +14,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse +from tests.unit.conftest import AsyncOnlyImageFetch _STATE: Final = "Customer asked for a refund" _SYSTEMONE_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( @@ -158,3 +159,330 @@ async def test_router_asystemone_logs_the_systemone_call_type( assert isinstance(response, DecisionsResponse) assert recorder.call_types == ["asystemone"] assert not isinstance(response, OpenAIDecisionResponse) + + +_BOX_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.14.0", + "answers": { + "has_mug": {"type": "noul", "noul": 0.97}, + "mug": {"type": "box", "box": {"top": 0.1, "bottom": 0.5, "left": 0.2, "right": 0.6}, "confidence": 0.84}, + }, + "usage": {"input_tokens": 299, "output_tokens": 37}, +} +_MUG_BOX: Final[Mapping[str, float]] = MappingProxyType({"top": 0.1, "bottom": 0.5, "left": 0.2, "right": 0.6}) +_IMAGES_HEADER: Final[Mapping[str, str]] = MappingProxyType({"TypeSafe-Danger-Zone-Images": "true"}) + + +@pytest.mark.asyncio +async def test_systemone_sends_assets_and_box_questions_to_typesafe(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + questions: Final[Mapping[str, object]] = { + "has_mug": {"type": "noul", "instructions": "Is there a mug in `photo-1`?"}, + "mug": {"type": "box", "instructions": "the mug in `photo-1`"}, + } + + response: Final = await litellm.asystemone( + model="typesafe/jev-1.14.0", + state={"photo": "photo-1"}, + assets={"photo-1": {"type": "image", "format": "png", "data_base64": "AA=="}}, + questions=questions, + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "jev-1.14.0", + "state": {"photo": "photo-1"}, + "questions": questions, + "assets": {"photo-1": {"type": "image", "format": "png", "data_base64": "AA=="}}, + } + assert isinstance(response, DecisionsResponse) + assert response.answers["mug"].model_dump(mode="json") == {"type": "box", "box": _MUG_BOX, "confidence": 0.84} + + +@pytest.mark.asyncio +async def test_decisions_input_images_reach_typesafe_as_assets_referenced_in_the_state( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + response: Final = await litellm.adecisions( + model="typesafe/jev-1.14.0", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Desk photo:"}, + {"type": "input_image", "image_url": _PNG_DATA_URL}, + ], + } + ], + questions=[ + {"type": "predicate", "name": "has_mug", "instructions": "Is there a mug on the desk?"}, + {"type": "box", "name": "mug", "instructions": "the mug on the desk"}, + ], + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + ) + + sent: Final = json.loads(route.calls.last.request.content) + (reference,) = sent["assets"] + assert sent["state"] == f"Desk photo:\n\n{reference}" + assert sent["assets"][reference] == {"type": "image", "format": "png", "data_base64": "AA=="} + assert sent["questions"]["mug"] == {"type": "box", "instructions": "the mug on the desk"} + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "has_mug", "probability": 0.97}, + {"type": "box", "name": "mug", "box": _MUG_BOX, "confidence": 0.84}, + ] + + +@pytest.mark.asyncio +async def test_decisions_remote_images_reach_typesafe_inlined_as_assets( + respx_mock: respx.MockRouter, async_only_image_fetch: AsyncOnlyImageFetch +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + await litellm.adecisions( + model="typesafe/jev-1.14.0", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": "https://images.example/desk.png"}]}], + questions=[{"type": "predicate", "name": "has_mug", "instructions": "Is there a mug on the desk?"}], + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert async_only_image_fetch.fetched == ["https://images.example/desk.png"] + assert sent["assets"] == { + sent["state"]: {"type": "image", "format": "png", "data_base64": async_only_image_fetch.base64_png} + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scoped_provider", "expected_headers"), + (("typesafe", {"typesafe-danger-zone-long-context": "true"}), ("perplexity", {})), + ids=("typesafe", "another_provider"), +) +async def test_provider_specific_headers_reach_typesafe_only_when_scoped_to_it( + scoped_provider: str, expected_headers: Mapping[str, str], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + await litellm.asystemone( + model="typesafe/jev-1.14.0", + state=_STATE, + questions=_SYSTEMONE_QUESTIONS, + api_key="caller-key", + provider_specific_header={ + "custom_llm_provider": scoped_provider, + "extra_headers": {"TypeSafe-Danger-Zone-Long-Context": "true"}, + }, + ) + + sent_headers: Final = route.calls.last.request.headers + assert { + name: value for name, value in sent_headers.items() if name.startswith("typesafe-danger-zone-") + } == expected_headers + + +_DESK_PHOTO_INPUT: Final[tuple[Mapping[str, object], ...]] = ( + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Desk photo:"}, + {"type": "input_image", "image_url": _PNG_DATA_URL}, + ], + }, +) +_HAS_MUG_QUESTION: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "has_mug", "instructions": "Is there a mug on the desk?"}, +) +_WITHOUT_IMAGES_HEADER: Final[tuple[Mapping[str, str], ...]] = ({}, {"TypeSafe-Danger-Zone-Images": "false"}) + + +@pytest.mark.parametrize("extra_headers", _WITHOUT_IMAGES_HEADER, ids=("no_header", "header_false")) +def test_typesafe_refuses_images_and_assets_before_http_without_the_images_header( + extra_headers: Mapping[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['input_image'\]") as image_refusal: + litellm.decisions( + model="typesafe/jev-1.14.0", + input=_DESK_PHOTO_INPUT, + questions=_HAS_MUG_QUESTION, + api_key="caller-key", + extra_headers=extra_headers, + ) + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['assets'\]") as asset_refusal: + litellm.systemone( + model="typesafe/jev-1.14.0", + state="review photo-1", + assets={"photo-1": {"format": "png", "data_base64": "AA=="}}, + questions=_SYSTEMONE_QUESTIONS, + api_key="caller-key", + extra_headers=extra_headers, + ) + + assert (image_refusal.value.status_code, asset_refusal.value.status_code) == (400, 400) + assert not route.called + + +@pytest.mark.asyncio +async def test_typesafe_drops_images_and_assets_without_the_images_header_under_drop_params( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + await litellm.adecisions( + model="typesafe/jev-1.14.0", + input=_DESK_PHOTO_INPUT, + questions=_HAS_MUG_QUESTION, + api_key="caller-key", + drop_params=True, + ) + decisions_body: Final = json.loads(route.calls.last.request.content) + await litellm.asystemone( + model="typesafe/jev-1.14.0", + state="review photo-1", + assets={"photo-1": {"format": "png", "data_base64": "AA=="}}, + questions=_SYSTEMONE_QUESTIONS, + api_key="caller-key", + drop_params=True, + ) + systemone_body: Final = json.loads(route.calls.last.request.content) + + assert (decisions_body["state"], "assets" in decisions_body) == ("Desk photo:", False) + assert "assets" not in systemone_body + + +_BOX_UNSUPPORTED: Final[tuple[tuple[str, Mapping[str, str]], ...]] = ( + ("typesafe/jev-1.13.0", _IMAGES_HEADER), + ("typesafe/jev-1.14.0", MappingProxyType({})), +) +_BOX_UNSUPPORTED_IDS: Final = ("model_without_box_support", "without_the_images_header") + + +@pytest.mark.parametrize(("model", "extra_headers"), _BOX_UNSUPPORTED, ids=_BOX_UNSUPPORTED_IDS) +def test_typesafe_refuses_box_questions_before_http_when_the_model_or_the_headers_do_not_allow_them( + model: str, extra_headers: Mapping[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]") as caught: + litellm.systemone( + model=model, + state="a mug on a desk", + questions={"mug": {"type": "box", "instructions": "the mug"}, **_SYSTEMONE_QUESTIONS}, + api_key="caller-key", + extra_headers=extra_headers, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "extra_headers"), _BOX_UNSUPPORTED, ids=_BOX_UNSUPPORTED_IDS) +async def test_typesafe_answers_box_questions_it_cannot_take_as_refusals_under_drop_params( + model: str, extra_headers: Mapping[str, str], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + response: Final = await litellm.adecisions( + model=model, + input="Customer asked for a refund", + questions=(*_OPENAI_QUESTIONS, {"type": "box", "name": "mug", "instructions": "the mug"}), + api_key="caller-key", + extra_headers=extra_headers, + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["questions"] == { + "is_refund": {"type": "noul", "instructions": "Is this a refund request?"} + } + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_refund", "probability": 0.93}, + {"type": "refusal", "name": "mug"}, + ] + + +def test_a_malformed_provider_specific_header_is_ignored(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + response: Final = litellm.systemone( + model="typesafe/jev-1.14.0", + state=_STATE, + questions=_SYSTEMONE_QUESTIONS, + api_key="caller-key", + provider_specific_header={"custom_llm_provider": "typesafe", "extra_headers": "not a header map"}, + ) + + assert isinstance(response, DecisionsResponse) + assert not [name for name in route.calls.last.request.headers if name.startswith("typesafe-danger-zone-")] + + +@pytest.mark.asyncio +async def test_url_safe_base64_images_reach_typesafe_as_standard_base64_assets(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + await litellm.adecisions( + model="typesafe/jev-1.14.0", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,-_8="}]}], + questions=_HAS_MUG_QUESTION, + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + ) + + assert list(json.loads(route.calls.last.request.content)["assets"].values()) == [ + {"type": "image", "format": "png", "data_base64": "+/8="} + ] + + +def test_an_image_whose_data_url_is_not_base64_answers_400_before_typesafe_is_called( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_BOX_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="input_image") as caught: + litellm.decisions( + model="typesafe/jev-1.14.0", + input=[ + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,not base64!"}]} + ], + questions=_HAS_MUG_QUESTION, + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +async def test_malformed_assets_are_dropped_and_well_formed_ones_kept_under_drop_params( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + await litellm.asystemone( + model="typesafe/jev-1.14.0", + state="compare photo-1 with photo-2", + assets={ + "photo-1": {"type": "image", "format": "png", "data_base64": "AA=="}, + "photo-2": {"type": "image", "part": "file1"}, + }, + questions=_SYSTEMONE_QUESTIONS, + api_key="caller-key", + extra_headers=_IMAGES_HEADER, + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content)["assets"] == { + "photo-1": {"type": "image", "format": "png", "data_base64": "AA=="} + } diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f50be89d032..768f8b7bf59 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1135,6 +1135,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_tool_choice": {"type": "boolean"}, "supports_tool_search": {"type": "boolean"}, "supports_video_input": {"type": "boolean"}, + "supports_box_questions": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 53e07c3814a..df2d7dc64f5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4713,8 +4713,9 @@ export interface paths { * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with - * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these - * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the + * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before + * any provider is called. `model` is any decision model in the proxy * model list. System One decision models accept this format too, LiteLLM translates the request and the * answers. Streaming is not supported. * @@ -15943,9 +15944,10 @@ export interface paths { * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. * * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with - * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, - * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider - * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`). + * TypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break + * these rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) @@ -19849,8 +19851,9 @@ export interface paths { * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with - * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these - * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the + * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before + * any provider is called. `model` is any decision model in the proxy * model list. System One decision models accept this format too, LiteLLM translates the request and the * answers. Streaming is not supported. * @@ -22225,9 +22228,10 @@ export interface paths { * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. * * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with - * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, - * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider - * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`). + * TypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break + * these rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) @@ -27542,6 +27546,19 @@ export interface components { /** Input Reference */ input_reference?: string | null; }; + /** BoxAnswer */ + BoxAnswer: { + box: components["schemas"]["DecisionsBox"]; + /** Confidence */ + confidence: number | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "box"; + } & { + [key: string]: unknown; + }; /** * BreakdownMetrics * @description Breakdown of spend by different dimensions @@ -31354,11 +31371,24 @@ export interface components { */ threshold: number; }; + /** DecisionsBox */ + DecisionsBox: { + /** Bottom */ + bottom: number; + /** Left */ + left: number; + /** Right */ + right: number; + /** Top */ + top: number; + } & { + [key: string]: unknown; + }; /** DecisionsResponse */ DecisionsResponse: { /** Answers */ answers: { - [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"]; + [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"] | components["schemas"]["BoxAnswer"]; }; /** Model */ model?: string | null; @@ -39343,6 +39373,19 @@ export interface components { [key: string]: unknown; } | null; }; + /** OpenAIBoxAnswer */ + OpenAIBoxAnswer: { + box: components["schemas"]["DecisionsBox"]; + /** Confidence */ + confidence: number | null; + /** Name */ + name: string | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "box"; + }; /** OpenAIChoiceAnswer */ OpenAIChoiceAnswer: { /** Choice */ @@ -39390,7 +39433,7 @@ export interface components { /** OpenAIDecisionResponse */ OpenAIDecisionResponse: { /** Answers */ - answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIBoxAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; /** Model */ model: string; usage: components["schemas"]["OpenAIDecisionUsage"]; @@ -57521,6 +57564,22 @@ export interface operations { * @constant */ type: "score"; + } | { + /** + * Instructions + * @description What to find in the input's image + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "box"; })[]; /** * Safety Identifier @@ -71788,6 +71847,27 @@ export interface operations { * } */ "application/json": { + /** + * Assets + * @description TypeSafe only. Images keyed by a reference string. Write the reference in state, instructions or criteria where the image belongs + */ + assets?: { + [key: string]: { + /** + * Data Base64 + * @description Base64-encoded image bytes, no data: prefix + */ + data_base64: string; + /** + * Format + * @description The image's format, in lowercase + * @enum {string} + */ + format: "png" | "jpg" | "jpeg"; + /** Type */ + type?: "image" | null; + }; + } | null; /** * Audio * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} @@ -71901,6 +71981,21 @@ export interface operations { type: "score"; } & { [key: string]: unknown; + }) | ({ + /** + * Instructions + * @description What to find, and in which image, such as 'the coffee mug in `photo`' + */ + instructions: string | { + [key: string]: unknown; + } | unknown[]; + /** + * Type + * @constant + */ + type: "box"; + } & { + [key: string]: unknown; }); }; /** @@ -77418,6 +77513,22 @@ export interface operations { * @constant */ type: "score"; + } | { + /** + * Instructions + * @description What to find in the input's image + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "box"; })[]; /** * Safety Identifier @@ -80668,6 +80779,27 @@ export interface operations { * } */ "application/json": { + /** + * Assets + * @description TypeSafe only. Images keyed by a reference string. Write the reference in state, instructions or criteria where the image belongs + */ + assets?: { + [key: string]: { + /** + * Data Base64 + * @description Base64-encoded image bytes, no data: prefix + */ + data_base64: string; + /** + * Format + * @description The image's format, in lowercase + * @enum {string} + */ + format: "png" | "jpg" | "jpeg"; + /** Type */ + type?: "image" | null; + }; + } | null; /** * Audio * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} @@ -80781,6 +80913,21 @@ export interface operations { type: "score"; } & { [key: string]: unknown; + }) | ({ + /** + * Instructions + * @description What to find, and in which image, such as 'the coffee mug in `photo`' + */ + instructions: string | { + [key: string]: unknown; + } | unknown[]; + /** + * Type + * @constant + */ + type: "box"; + } & { + [key: string]: unknown; }); }; /**