diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 9c0e51e14ac..c44bc8505e4 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,7 +1,7 @@ import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace -from functools import partial +from functools import partial, reduce from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias @@ -35,7 +35,11 @@ from litellm.llms.base_llm.decisions.transformation import ( with_inlined_images, ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig, + ir_to_openai_response, + openai_request_to_ir, +) from litellm.types.decisions import ( DecisionQuestion, DecisionsAsset, @@ -56,15 +60,18 @@ from litellm.types.decisions import ( DecisionsRequestBody, DecisionsResponse, DecisionsVideo, + OpenAIDecisionAnswer, OpenAIDecisionInput, OpenAIDecisionQuestion, OpenAIDecisionRequestBody, OpenAIDecisionResponse, + OpenAIRefusalAnswer, UnsupportedDecisionsRequest, ) from litellm.types.llms.openai import ChatCompletionUserMessage +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode -from litellm.utils import ProviderConfigManager, get_model_cost_key +from litellm.utils import ProviderConfigManager, filter_out_litellm_params, get_model_cost_key SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] @@ -76,6 +83,9 @@ DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequest _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( + OpenAIDecisionQuestions | None +) _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( str | None, config=ConfigDict(title="safety_identifier") ) @@ -86,18 +96,19 @@ _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None] ] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None) _HANDLER: Final = BaseLLMHTTPHandler() +_LITELLM_SETTINGS: Final = frozenset((*GenericLiteLLMParams.model_fields, "base_url", "additional_drop_params", "user")) _BASE64_MARKER: Final = ";base64," _IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images")) -_IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) _ASSETS_ADAPTER: Final[TypeAdapter[DecisionsAssets | None]] = TypeAdapter( DecisionsAssets | None, config=ConfigDict(title="assets") ) _ASSET_ENTRY_ADAPTER: Final[TypeAdapter[tuple[DecisionsAssetReference, DecisionsAsset]]] = TypeAdapter( tuple[DecisionsAssetReference, DecisionsAsset] ) -_ASSET_ENTRIES_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) def _valid_image(image: object) -> DecisionsImage | None: @@ -107,9 +118,9 @@ def _valid_image(image: object) -> DecisionsImage | None: return None -def _image_entries(images: object) -> tuple[object, ...]: +def _entries(value: object) -> tuple[object, ...]: try: - return _IMAGE_ENTRIES_ADAPTER.validate_python(images) + return _ENTRIES_ADAPTER.validate_python(value) except ValidationError: return () @@ -129,9 +140,9 @@ def _valid_asset(entry: tuple[str, object]) -> tuple[str, DecisionsAsset] | None return None -def _asset_entries(assets: object) -> Mapping[str, object]: +def _fields(value: object) -> Mapping[str, object]: try: - return _ASSET_ENTRIES_ADAPTER.validate_python(assets) + return _FIELDS_ADAPTER.validate_python(value) except ValidationError: return MappingProxyType({}) @@ -140,7 +151,7 @@ def _request_assets(assets: object, drops_params: bool) -> Mapping[str, Decision if not drops_params: return _ASSETS_ADAPTER.validate_python(assets) valid: Final = MappingProxyType( - dict(entry for entry in map(_valid_asset, _asset_entries(assets).items()) if entry is not None) + dict(entry for entry in map(_valid_asset, _fields(assets).items()) if entry is not None) ) return valid or None @@ -148,7 +159,7 @@ def _request_assets(assets: object, drops_params: bool) -> Mapping[str, Decision def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]: if not drops_params: return _IMAGES_ADAPTER.validate_python(images) or () - return tuple(image for image in map(_valid_image, _image_entries(images)) if image is not None) + return tuple(image for image in map(_valid_image, _entries(images)) if image is not None) @dataclass(frozen=True, slots=True) @@ -176,7 +187,7 @@ class SystemOneRequestFields: @dataclass(frozen=True, slots=True) class OpenAIDecisionRequestFields: input: OpenAIDecisionInput | None - questions: OpenAIDecisionQuestions | None + questions: object audio: DecisionsAudioClips | None = None videos: DecisionsVideoClips | None = None @@ -212,6 +223,28 @@ class DecisionsCall: caller_questions: tuple[DecisionsIRQuestion, ...] +@dataclass(frozen=True, slots=True) +class DroppedQuestion: + position: int + name: str | None + + +@dataclass(frozen=True, slots=True, repr=False) +class NativeOpenAIDecisionsCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: OpenAIDecisionsConfig + body: Mapping[str, object] + dropped_questions: tuple[DroppedQuestion, ...] + api_base: str + api_key: str | None = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] = field(repr=False) + + def supported_decisions_providers() -> tuple[str, ...]: return tuple( provider.value @@ -505,18 +538,23 @@ def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> logging_obj.update_messages([_logged_request_message(caller_request)]) -def prepare_call( - *, +@dataclass(frozen=True, slots=True, repr=False) +class _Route: + requested_model: str + model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + api_base: str | None + api_key: str | None = field(repr=False) + + +def _route( model: str, - fields: RequestFields, - safety_identifier: str | None, - api_key: str | None, - api_base: str | None, - timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, - extra_headers: Mapping[str, str] | None, + api_base: str | None, + api_key: str | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> _Route: base_url_alias: Final = kwargs.get("base_url") call_api_base: Final = ( base_url_alias if api_base is None and isinstance(base_url_alias, str) and base_url_alias else api_base @@ -538,40 +576,118 @@ def prepare_call( canonical_model: Final = provider_config.canonical_model(upstream_model) except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error + return _Route( + requested_model=model, + model=canonical_model, + custom_llm_provider=provider, + provider_config=provider_config, + api_base=dynamic_api_base or call_api_base, + api_key=dynamic_api_key or api_key, + ) + + +def _invalid_request(error: ValueError | TypeError, route: _Route) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _connection(route: _Route, kwargs: Mapping[str, object]) -> tuple[str, str | None]: try: - headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs) - support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields) + resolved_api_base, static_api_key = route.provider_config.resolve_connection(route.api_base, route.api_key) + except (ValueError, httpx.InvalidURL) as error: + raise litellm.BadRequestError( + message=str(error), model=route.requested_model, llm_provider=route.custom_llm_provider + ) from error + if resolved_api_base is None: + raise litellm.BadRequestError( + message=route.provider_config.missing_api_base_message(route.custom_llm_provider), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + resolved_api_key: Final = route.provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs) + if resolved_api_key is None and route.provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{route.custom_llm_provider}'", + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + return resolved_api_base, resolved_api_key + + +def _unsupported_params_error(names: Sequence[str], route: _Route) -> litellm.UnsupportedParamsError: + return litellm.UnsupportedParamsError( + message=( + f"{route.custom_llm_provider} does not support parameters: {list(names)}, " + f"for model={route.requested_model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _nothing_left_error(nothing_left: _NothingLeftAfterDrop, route: _Route) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=( + f"{route.custom_llm_provider} does not support parameters: {list(nothing_left.names)}, " + f"for model={route.requested_model}, and dropping them leaves no {nothing_left.missing}" + ), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _call_logging_obj(route: _Route, api_base: str, kwargs: Mapping[str, object]) -> LiteLLMLoggingObj | None: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLoggingObj): + return None + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=route.model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": route.provider_config.get_complete_url(api_base, route.model), + "input_cost_per_query": ( + kwargs.get("input_cost_per_query") + if route.custom_llm_provider in QUERY_PRICED_DECISIONS_PROVIDERS + else None + ), + }, + custom_llm_provider=route.custom_llm_provider, + ) + return logging_obj + + +def _ir_call( + route: _Route, + fields: RequestFields, + safety_identifier: str | None, + timeout: float | httpx.Timeout | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + provider_config: Final = route.provider_config + checked_fields: Final = ( + replace(fields, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(fields.questions)) + if isinstance(fields, OpenAIDecisionRequestFields) + else fields + ) + try: + headers: Final = decisions_outbound_headers(extra_headers, route.custom_llm_provider, kwargs) + support: Final = _resolved_support( + provider_config, route.model, route.custom_llm_provider, headers, kwargs, checked_fields + ) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) - request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate( + request: Final = _fields_without_unsupported_media(checked_fields, support, kwargs).validate( safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs) ) except ValidationError as error: - raise litellm.BadRequestError( - message=f"Invalid Decisions request: {error}", - model=model, - llm_provider=provider, - ) from error - - try: - resolved_api_base, static_api_key = provider_config.resolve_connection( - dynamic_api_base or call_api_base, dynamic_api_key or api_key - ) - except (ValueError, httpx.InvalidURL) as error: - raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error - if resolved_api_base is None: - raise litellm.BadRequestError( - message=provider_config.missing_api_base_message(provider), - model=model, - llm_provider=provider, - ) - resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs) - if resolved_api_key is None and provider_config.api_key_required: - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, - ) - + raise _invalid_request(error, route) from error + resolved_api_base, resolved_api_key = _connection(route, kwargs) caller_ir_request: Final = _without_malformed_images( _ir_request(request, request_safety_identifier), _drops_params(kwargs) ) @@ -581,55 +697,26 @@ def prepare_call( f"Invalid Decisions request: {caller_ir_request.count} input_image part(s) have an image_url that is " "neither a data:image/...;base64, URL nor an http(s) URL" ), - model=model, - llm_provider=provider, + model=route.requested_model, + llm_provider=route.custom_llm_provider, ) ir_request: Final = _provider_ir_request( caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs ) if isinstance(ir_request, _UnsupportedParams): - raise litellm.UnsupportedParamsError( - message=( - f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}. " - "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" - ), - model=model, - llm_provider=provider, - ) + raise _unsupported_params_error(ir_request.names, route) if isinstance(ir_request, _NothingLeftAfterDrop): - raise litellm.BadRequestError( - message=( - f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, " - f"and dropping them leaves no {ir_request.missing}" - ), - model=model, - llm_provider=provider, - ) - logging_obj: Final = kwargs.get("litellm_logging_obj") - if isinstance(logging_obj, LiteLLMLoggingObj): - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=canonical_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), - "input_cost_per_query": ( - kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None - ), - }, - custom_llm_provider=provider, - ) + raise _nothing_left_error(ir_request, route) return DecisionsCall( - model=canonical_model, - requested_model=model, - custom_llm_provider=provider, + model=route.model, + requested_model=route.requested_model, + custom_llm_provider=route.custom_llm_provider, provider_config=provider_config, request=request, ir_request=ir_request, api_base=resolved_api_base, api_key=resolved_api_key, - logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + logging_obj=_call_logging_obj(route, resolved_api_base, kwargs), headers=headers, timeout=timeout, litellm_params=kwargs, @@ -637,6 +724,153 @@ def prepare_call( ) +@dataclass(frozen=True, slots=True) +class _NativeQuestions: + sent: object + dropped: tuple[DroppedQuestion, ...] + + +def _question_name(question: object) -> str | None: + name: Final = _fields(question).get("name") + return name if isinstance(name, str) else None + + +def _native_questions( + questions: object, support: _ResolvedSupport, fields: OpenAIDecisionRequestFields, kwargs: Mapping[str, object] +) -> _NativeQuestions | _UnsupportedParams | _NothingLeftAfterDrop: + entries: Final = _entries(questions) + box_positions: Final = frozenset( + position for position, question in enumerate(entries) if _fields(question).get("type") == "box" + ) + refuses_box_questions: Final = bool(box_positions) and not support.box_questions + unsupported: Final = ( + *(("box questions",) if refuses_box_questions else ()), + *(("audio",) if fields.audio and not support.audio else ()), + *(("videos",) if fields.videos and not support.videos else ()), + ) + if not unsupported: + return _NativeQuestions(sent=questions, dropped=()) + if not _drops_params(kwargs): + return _UnsupportedParams(names=unsupported) + if not refuses_box_questions: + return _NativeQuestions(sent=questions, dropped=()) + kept: Final = tuple(question for position, question in enumerate(entries) if position not in box_positions) + if not kept: + return _NothingLeftAfterDrop(names=unsupported, missing="questions to answer") + return _NativeQuestions( + sent=kept, + dropped=tuple( + DroppedQuestion(position=position, name=_question_name(entries[position])) + for position in sorted(box_positions) + ), + ) + + +def _plain_mapping(value: object) -> Mapping[str, object]: + if isinstance(value, Mapping): + return _FIELDS_ADAPTER.validate_python(value) + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def _sent_fields( + route: _Route, fields: OpenAIDecisionRequestFields, kwargs: Mapping[str, object] +) -> Mapping[str, object]: + explicit: Final = ( + ("input", fields.input), + ("questions", fields.questions), + ("audio", fields.audio), + ("videos", fields.videos), + ) + sent: Final = MappingProxyType( + { + **{name: value for name, value in explicit if value is not None}, + **filter_out_litellm_params(kwargs=kwargs, excluding=_LITELLM_SETTINGS), + } + ) + try: + return _FIELDS_ADAPTER.validate_python(to_jsonable_python(sent, exclude_none=True, fallback=_plain_mapping)) + except (TypeError, ValueError) as error: + raise _invalid_request(error, route) from error + + +def _native_body( + model: str, sent: Mapping[str, object], questions: object, safety_identifier: str | None, support: _ResolvedSupport +) -> Mapping[str, object]: + unsupported_media: Final = frozenset( + name for name, supported in (("audio", support.audio), ("videos", support.videos)) if not supported + ) + return MappingProxyType( + { + "model": model, + **{name: value for name, value in sent.items() if name not in unsupported_media}, + **({} if questions is None else {"questions": questions}), + **({} if safety_identifier is None else {"safety_identifier": safety_identifier}), + } + ) + + +def _native_openai_call( + route: _Route, + provider_config: OpenAIDecisionsConfig, + fields: OpenAIDecisionRequestFields, + safety_identifier: str | None, + timeout: float | httpx.Timeout | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> NativeOpenAIDecisionsCall: + try: + headers: Final = decisions_outbound_headers(extra_headers, route.custom_llm_provider, kwargs) + support: Final = _resolved_support( + provider_config, route.model, route.custom_llm_provider, headers, kwargs, fields + ) + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) + except ValidationError as error: + raise _invalid_request(error, route) from error + resolved_api_base, resolved_api_key = _connection(route, kwargs) + sent: Final = _sent_fields(route, fields, kwargs) + questions: Final = _native_questions(sent.get("questions"), support, fields, kwargs) + if isinstance(questions, _UnsupportedParams): + raise _unsupported_params_error(questions.names, route) + if isinstance(questions, _NothingLeftAfterDrop): + raise _nothing_left_error(questions, route) + return NativeOpenAIDecisionsCall( + model=route.model, + requested_model=route.requested_model, + custom_llm_provider=route.custom_llm_provider, + provider_config=provider_config, + body=_native_body( + provider_config.request_model(route.model), sent, questions.sent, request_safety_identifier, support + ), + dropped_questions=questions.dropped, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=_call_logging_obj(route, resolved_api_base, kwargs), + headers=headers, + timeout=timeout, + litellm_params=kwargs, + ) + + +def prepare_call( + *, + model: str, + fields: RequestFields, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall | NativeOpenAIDecisionsCall: + route: Final = _route(model, custom_llm_provider, api_base, api_key, kwargs) + if isinstance(fields, OpenAIDecisionRequestFields) and isinstance(route.provider_config, OpenAIDecisionsConfig): + return _native_openai_call( + route, route.provider_config, fields, safety_identifier, timeout, extra_headers, kwargs + ) + return _ir_call(route, fields, safety_identifier, timeout, extra_headers, kwargs) + + def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]: if not call.provider_config.inlines_remote_images: return () @@ -753,8 +987,8 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: return body -def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str: - returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") +def _pricing_model(returned_model: str | None, call: DecisionsCall | NativeOpenAIDecisionsCall) -> str: + returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{returned_model}") if returned is not None and cost_map_key_prices_anything(returned): return returned return f"{call.custom_llm_provider}/{call.model}" @@ -767,7 +1001,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci { "model": f"{call.custom_llm_provider}/{call.model}", "custom_llm_provider": call.custom_llm_provider, - "provider_response_model": _pricing_model(response, call), + "provider_response_model": _pricing_model(response.model, call), **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), } ) @@ -798,7 +1032,7 @@ def _formatted_response( assert_never(call.request) -def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: +def _map_upstream_exception(error: Exception, call: DecisionsCall | NativeOpenAIDecisionsCall) -> Exception: provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" model: Final = f"{call.custom_llm_provider}/{call.model}" match error: @@ -816,7 +1050,7 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: ) -async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: +async def _asend_ir(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported))) body: Final = _request_body(inlined) @@ -840,7 +1074,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon return _format_response(response, inlined) -def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: +def _send_ir(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported))) body: Final = _request_body(inlined) @@ -864,6 +1098,90 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: return _format_response(response, inlined) +def _with_refusal( + answers: tuple[OpenAIDecisionAnswer, ...], dropped: DroppedQuestion +) -> tuple[OpenAIDecisionAnswer, ...]: + return (*answers[: dropped.position], OpenAIRefusalAnswer(name=dropped.name), *answers[dropped.position :]) + + +def _format_native_response( + response: OpenAIDecisionResponse, call: NativeOpenAIDecisionsCall +) -> OpenAIDecisionResponse: + formatted: Final = response.model_copy( + update={"answers": reduce(_with_refusal, call.dropped_questions, response.answers)} + ) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": _pricing_model(response.model, call), + } + ) + return formatted + + +async def _asend_native(call: NativeOpenAIDecisionsCall) -> OpenAIDecisionResponse: + try: + raw_response: Final = await _HANDLER.apost_decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + logged_body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + response: Final = call.provider_config.parse_native_response(call.custom_llm_provider, raw_response) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_native_response(response, call) + + +def _send_native(call: NativeOpenAIDecisionsCall) -> OpenAIDecisionResponse: + try: + raw_response: Final = _HANDLER.post_decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + logged_body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + response: Final = call.provider_config.parse_native_response(call.custom_llm_provider, raw_response) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_native_response(response, call) + + +async def asend(call: DecisionsCall | NativeOpenAIDecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + match call: + case NativeOpenAIDecisionsCall(): + return await _asend_native(call) + case DecisionsCall(): + return await _asend_ir(call) + case _: + assert_never(call) + + +def send(call: DecisionsCall | NativeOpenAIDecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + match call: + case NativeOpenAIDecisionsCall(): + return _send_native(call) + case DecisionsCall(): + return _send_ir(call) + case _: + assert_never(call) + + def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None: if field_name not in kwargs: return diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 4e609fd2ab8..c39f1e294fb 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -11,6 +11,7 @@ from litellm.decisions.call import ( DecisionsAudioClips, DecisionsCall, DecisionsVideoClips, + NativeOpenAIDecisionsCall, OpenAIDecisionQuestions, OpenAIDecisionRequestFields, RequestFields, @@ -27,9 +28,6 @@ from litellm.utils import client _STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None) _SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None) -_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( - OpenAIDecisionQuestions | None -) _SYSTEMONE_FIELDS: Final = frozenset({"state", "images", "assets"}) @@ -45,7 +43,7 @@ def _request_fields( reject_images_next_to_input(model=model, kwargs=kwargs) return OpenAIDecisionRequestFields( input=decision_input, - questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions), + questions=questions, audio=audio, videos=videos, ) @@ -78,7 +76,7 @@ def _prepare_decisions_call( custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> DecisionsCall | NativeOpenAIDecisionsCall: log_caller_request( kwargs, { diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 254783edf39..2db8a4ff0da 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -431,6 +431,14 @@ class DecisionsResponseValidationError(BaseLLMException): pass +def unexpected_response_error(custom_llm_provider: str, error: ValidationError) -> DecisionsResponseValidationError: + return DecisionsResponseValidationError( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " + f"{error.title} {_validation_summary(error)}", + ) + + def _lacks_confidence(answer: DecisionsIRAnswer) -> bool: return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None @@ -690,11 +698,7 @@ class BaseDecisionsConfig(ABC): payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) response: Final = self.parse_response(payload, request) except ValidationError as error: - raise DecisionsResponseValidationError( - status_code=500, - message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " - f"{error.title} {_validation_summary(error)}", - ) from error + raise unexpected_response_error(custom_llm_provider, error) from error if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers): raise DecisionsResponseValidationError( status_code=500, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2f17dc6dd10..7b72bdfecb7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1751,13 +1751,12 @@ class BaseLLMHTTPHandler: ) return url, outbound_headers, data - def decisions( + def post_decisions( self, model: str, custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsIRRequest, body: Mapping[str, object], logged_body: Mapping[str, object], api_base: str, @@ -1766,7 +1765,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None, litellm_params: Mapping[str, object], client: HTTPHandler | None = None, - ) -> DecisionsIRResponse: + ) -> httpx.Response: url, outbound_headers, data = self._prepare_decisions_request( model=model, logging_obj=logging_obj, @@ -1788,7 +1787,7 @@ class BaseLLMHTTPHandler: ) sync_httpx_client: Final = client if client is not None else get_httpx_client() try: - response: Final = sync_httpx_client.post( + return sync_httpx_client.post( url, content=signed_body, json=dict(data) if signed_body is None else None, @@ -1798,17 +1797,13 @@ class BaseLLMHTTPHandler: ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) - return provider_config.transform_decisions_response( - model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request - ) - async def adecisions( + async def apost_decisions( self, model: str, custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsIRRequest, body: Mapping[str, object], logged_body: Mapping[str, object], api_base: str, @@ -1817,7 +1812,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None, litellm_params: Mapping[str, object], client: AsyncHTTPHandler | None = None, - ) -> DecisionsIRResponse: + ) -> httpx.Response: url, outbound_headers, data = await self._aprepare_decisions_request( model=model, logging_obj=logging_obj, @@ -1847,7 +1842,7 @@ class BaseLLMHTTPHandler: else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) ) try: - response: Final = await async_httpx_client.post( + return await async_httpx_client.post( url, content=signed_body, json=dict(data) if signed_body is None else None, @@ -1857,6 +1852,71 @@ class BaseLLMHTTPHandler: ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + logged_body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: HTTPHandler | None = None, + ) -> DecisionsIRResponse: + response: Final = self.post_decisions( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + logged_body=logged_body, + api_base=api_base, + api_key=api_key, + headers=headers, + timeout=timeout, + litellm_params=litellm_params, + client=client, + ) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + logged_body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> DecisionsIRResponse: + response: Final = await self.apost_decisions( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + logged_body=logged_body, + api_base=api_base, + api_key=api_key, + headers=headers, + timeout=timeout, + litellm_params=litellm_params, + client=client, + ) return provider_config.transform_decisions_response( model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 5aa72579d6a..b2587efd0a7 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,11 +1,16 @@ from collections.abc import Mapping, Sequence from typing import Final -from pydantic import TypeAdapter +import httpx +from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never import litellm -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + decisions_text, + unexpected_response_error, +) from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( @@ -374,3 +379,9 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig): def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) + + def parse_native_response(self, custom_llm_provider: str, raw_response: httpx.Response) -> OpenAIDecisionResponse: + try: + return _OPENAI_RESPONSE_ADAPTER.validate_json(raw_response.content) + except ValidationError as error: + raise unexpected_response_error(custom_llm_provider, error) from error diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5cc1c53d396..db68e324cf7 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10638,6 +10638,7 @@ "type": "object" }, "OpenAIBoxAnswer": { + "additionalProperties": true, "properties": { "box": { "$ref": "#/components/schemas/DecisionsBox" @@ -10680,6 +10681,7 @@ "type": "object" }, "OpenAIChoiceAnswer": { + "additionalProperties": true, "properties": { "choice": { "anyOf": [ @@ -10737,6 +10739,7 @@ "type": "object" }, "OpenAIChoiceProbability": { + "additionalProperties": true, "properties": { "probability": { "title": "Probability", @@ -10762,6 +10765,7 @@ "type": "object" }, "OpenAIDecisionInputTokensDetails": { + "additionalProperties": true, "properties": { "cache_write_tokens": { "default": 0, @@ -10778,6 +10782,7 @@ "type": "object" }, "OpenAIDecisionOutputTokensDetails": { + "additionalProperties": true, "properties": { "reasoning_tokens": { "default": 0, @@ -10793,16 +10798,6 @@ "properties": { "answers": { "items": { - "discriminator": { - "mapping": { - "box": "#/components/schemas/OpenAIBoxAnswer", - "choice": "#/components/schemas/OpenAIChoiceAnswer", - "predicate": "#/components/schemas/OpenAIPredicateAnswer", - "refusal": "#/components/schemas/OpenAIRefusalAnswer", - "score": "#/components/schemas/OpenAIScoreAnswer" - }, - "propertyName": "type" - }, "oneOf": [ { "$ref": "#/components/schemas/OpenAIPredicateAnswer" @@ -10818,6 +10813,9 @@ }, { "$ref": "#/components/schemas/OpenAIRefusalAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIUnknownAnswer" } ] }, @@ -10841,6 +10839,7 @@ "type": "object" }, "OpenAIDecisionUsage": { + "additionalProperties": true, "properties": { "input_tokens": { "title": "Input Tokens", @@ -10877,6 +10876,7 @@ "type": "object" }, "OpenAIPredicateAnswer": { + "additionalProperties": true, "properties": { "name": { "anyOf": [ @@ -10908,6 +10908,7 @@ "type": "object" }, "OpenAIRefusalAnswer": { + "additionalProperties": true, "properties": { "name": { "anyOf": [ @@ -10934,6 +10935,7 @@ "type": "object" }, "OpenAIScoreAnswer": { + "additionalProperties": true, "properties": { "confidence": { "anyOf": [ @@ -10984,6 +10986,7 @@ "type": "object" }, "OpenAIScoreProbability": { + "additionalProperties": true, "properties": { "label": { "title": "Label", @@ -11006,6 +11009,31 @@ "title": "OpenAIScoreProbability", "type": "object" }, + "OpenAIUnknownAnswer": { + "additionalProperties": true, + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "OpenAIUnknownAnswer", + "type": "object" + }, "ScoreAnswer": { "additionalProperties": true, "properties": { @@ -11070,7 +11098,7 @@ "paths": { "/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against one or more `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI\nand Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the\ndeployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM\nupdate. System One decision models accept this format too, LiteLLM translates the request and the answers and\nrejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", "requestBody": { "content": { @@ -12476,7 +12504,7 @@ }, "/v1/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against one or more `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI\nand Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the\ndeployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM\nupdate. System One decision models accept this format too, LiteLLM translates the request and the answers and\nrejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", "requestBody": { "content": { diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index a8684dd9757..1f34a9e8c64 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -4,7 +4,7 @@ from typing import Annotated, Final, Literal, TypeAlias from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth @@ -15,7 +15,6 @@ from litellm.types.decisions import ( DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequest, - OpenAIDecisionRequestBody, OpenAIDecisionResponse, ) from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, ExtractionResponse @@ -23,9 +22,15 @@ from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, E router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) -_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( - OpenAIDecisionRequestBody -) + + +class _OpenAIFormatBody(BaseModel): + state: None = None + + model_config = ConfigDict(extra="allow", frozen=True, title="OpenAI format Decisions request") + + +_OPENAI_FORMAT_BODY_ADAPTER: Final[TypeAdapter[_OpenAIFormatBody]] = TypeAdapter(_OpenAIFormatBody) _EXTRACTION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) @@ -139,7 +144,7 @@ async def _process_decisions( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, body_adapter: ( - TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody] | TypeAdapter[ExtractionRequestBody] + TypeAdapter[DecisionsRequestBody] | TypeAdapter[_OpenAIFormatBody] | TypeAdapter[ExtractionRequestBody] ), route_type: _RouteType, ) -> object: @@ -272,14 +277,15 @@ async def decisions( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ - Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + Judge an `input` against one or more `questions` and get the answers back in the same order. Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - any provider is called. `model` is any decision model in the proxy - model list. System One decision models accept this format too, LiteLLM translates the request and the - answers. Streaming is not supported. + `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + update. System One decision models accept this format too, LiteLLM translates the request and the answers and + rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. [Docs](https://docs.litellm.ai/docs/decisions) """ @@ -287,7 +293,7 @@ async def decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, + body_adapter=_OPENAI_FORMAT_BODY_ADAPTER, route_type="adecisions", ) diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 27fa84e3f9b..cb965676487 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -11,6 +11,7 @@ from litellm.decisions.call import ( DecisionsCall, DecisionsImages, DecisionsVideoClips, + NativeOpenAIDecisionsCall, SystemOneQuestions, SystemOneRequestFields, asend, @@ -39,7 +40,7 @@ def _prepare_systemone_call( custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> DecisionsCall | NativeOpenAIDecisionsCall: fields: Final = SystemOneRequestFields( state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 3b28225f0ff..5eb535369a1 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -3,7 +3,18 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias -from pydantic import AfterValidator, ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config +from pydantic import ( + AfterValidator, + ConfigDict, + Discriminator, + Field, + PrivateAttr, + Tag, + TypeAdapter, + field_validator, + model_validator, + with_config, +) from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -416,14 +427,14 @@ class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): name: str | None probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIChoiceProbability(LiteLLMPydanticObjectBase): value: str | bool probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): @@ -433,7 +444,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): probabilities: tuple[OpenAIChoiceProbability, ...] confidence: float | None = None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIScoreProbability(LiteLLMPydanticObjectBase): @@ -441,7 +452,7 @@ class OpenAIScoreProbability(LiteLLMPydanticObjectBase): label: str probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): @@ -451,14 +462,14 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): probabilities: tuple[OpenAIScoreProbability, ...] confidence: float | None = None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): type: Literal["refusal"] = "refusal" name: str | None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIBoxAnswer(LiteLLMPydanticObjectBase): @@ -467,12 +478,43 @@ class OpenAIBoxAnswer(LiteLLMPydanticObjectBase): box: DecisionsBox confidence: float | None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) + + +class OpenAIUnknownAnswer(LiteLLMPydanticObjectBase): + type: str + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +_KNOWN_OPENAI_ANSWER_TYPES: Final = frozenset({"predicate", "choice", "score", "box", "refusal"}) + + +class _TaggedAnswer(TypedDict, total=False): + type: ReadOnly[object] + + +_TAGGED_ANSWER_ADAPTER: Final[TypeAdapter[_TaggedAnswer]] = TypeAdapter(_TaggedAnswer) + + +def _openai_answer_tag(answer: object) -> str: + answer_type: Final = ( + _TAGGED_ANSWER_ADAPTER.validate_python(answer).get("type") + if isinstance(answer, Mapping) + else getattr(answer, "type", None) + ) + return answer_type if isinstance(answer_type, str) and answer_type in _KNOWN_OPENAI_ANSWER_TYPES else "unknown" OpenAIDecisionAnswer: TypeAlias = Annotated[ - OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIBoxAnswer | OpenAIRefusalAnswer, - Field(discriminator="type"), + Annotated[OpenAIPredicateAnswer, Tag("predicate")] + | Annotated[OpenAIChoiceAnswer, Tag("choice")] + | Annotated[OpenAIScoreAnswer, Tag("score")] + | Annotated[OpenAIBoxAnswer, Tag("box")] + | Annotated[OpenAIRefusalAnswer, Tag("refusal")] + | Annotated[OpenAIUnknownAnswer, Tag("unknown")], + Discriminator(_openai_answer_tag), ] @@ -480,13 +522,13 @@ class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): cached_tokens: int = 0 cache_write_tokens: int = 0 - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionOutputTokensDetails(LiteLLMPydanticObjectBase): reasoning_tokens: int = 0 - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): @@ -496,7 +538,7 @@ class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): output_tokens_details: OpenAIDecisionOutputTokensDetails = OpenAIDecisionOutputTokensDetails() total_tokens: int - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionResponse(_HiddenParamsResponse): diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py index 36a16dd5ad1..3b8db808208 100644 --- a/tests/unit/decisions/test_call.py +++ b/tests/unit/decisions/test_call.py @@ -7,6 +7,7 @@ from typing import Final import pytest import litellm +from litellm.decisions.call import NativeOpenAIDecisionsCall from litellm.decisions.main import _prepare_decisions_call from litellm.litellm_core_utils.litellm_logging import Logging from litellm.systemone.main import _prepare_systemone_call @@ -61,6 +62,43 @@ def test_systemone_logs_every_request_field_the_caller_sent(): assert set(logged) == {"state", "questions"} +def test_an_openai_body_carries_the_caller_fields_and_none_of_the_settings_litellm_owns( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + question: Final = {"type": "predicate", "instructions": QUESTION_TEXT, "rubric": "strict"} + + call: Final = _prepare_decisions_call( + **{**CONNECTION, "model": "openai/gpt-6-luna", "api_base": None}, + decision_input="My card was charged twice.", + questions=[question], + audio=None, + videos=None, + kwargs={ + "prompt_cache_key": "billing-v1", + "user": "key-hash", + "drop_params": True, + "organization": "org-1", + "base_url": "https://elsewhere.example/v1", + "additional_drop_params": ["temperature"], + "aws_region_name": "us-east-1", + "vertex_project": "project-1", + "azure_ad_token": "token-1", + "metadata": {"tags": ["billing"]}, + }, + ) + + assert isinstance(call, NativeOpenAIDecisionsCall) + assert call.body == { + "model": "gpt-6-luna", + "input": "My card was charged twice.", + "questions": [question], + "prompt_cache_key": "billing-v1", + } + + def test_decisions_logs_every_request_field_the_caller_sent(): logging_obj: Final = _logging_obj("adecisions") diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 789d59bd613..f459c863300 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -134,12 +134,13 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ) +_OPENAI_RESPONSE_ANSWERS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "sentiment"}, +) _OPENAI_RESPONSE: Final[Mapping[str, object]] = { "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "is_defect", "probability": 0.9}, - {"type": "refusal", "name": "sentiment"}, - ], + "answers": list(_OPENAI_RESPONSE_ANSWERS), "usage": { "input_tokens": _INPUT_TOKENS, "input_tokens_details": {"cached_tokens": _CACHED_TOKENS, "cache_write_tokens": _CACHE_WRITE_TOKENS}, @@ -2364,22 +2365,20 @@ def openai_default_base(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("OPENAI_API_BASE", raising=False) -@pytest.mark.usefixtures("cloudflare_account", "openai_default_base") -@pytest.mark.parametrize( - ("model", "upstream"), - (("cloudflare/clef", f"{_CLOUDFLARE_RUN_URL}/clef"), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)), - ids=("clef", "luna"), -) +@pytest.mark.usefixtures("cloudflare_account") @pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS) def test_malformed_input_image_urls_are_rejected_before_http( - model: str, upstream: str, image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(upstream).respond(json=_RESPONSE) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 1 input_image") as caught: litellm.decisions( - model=model, api_key="caller-key", input=_screen_messages(image_url), questions=_PREDICATE_QUESTIONS + model="cloudflare/clef", + api_key="caller-key", + input=_screen_messages(image_url), + questions=_PREDICATE_QUESTIONS, ) assert caught.value.status_code == 400 @@ -4085,22 +4084,196 @@ async def test_an_openai_format_deployment_that_advertises_media_is_still_refuse assert not route.called -def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None: - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) +@pytest.mark.usefixtures("openai_default_base") +@pytest.mark.parametrize( + ("model", "upstream"), + (("perplexity/pplx-decider-v1-27b", _PERPLEXITY_DECISIONS_URL), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)), + ids=("translated", "openai_native"), +) +def test_dropping_every_question_answers_400_before_http( + model: str, upstream: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(upstream).respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match="leaves no questions to answer") as caught: litellm.decisions( - model="perplexity/pplx-decider-v1-27b", + model=model, input="review", questions=[{"type": "box", "name": "crack", "instructions": "the crack on the screen"}], api_key="caller-key", drop_params=True, + model_info={"supports_box_questions": False}, ) assert caught.value.status_code == 400 assert not route.called +_NATIVE_OPENAI_INPUT: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "message", "role": "developer", "content": "Judge for the refunds team."}, + {"role": "user", "content": [{"type": "input_json", "json": {"order": 1234, "status": "damaged"}}]}, +) +_NATIVE_OPENAI_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?", "rubric": "visible damage only"}, + {"type": "ranking", "name": "remedy", "instructions": "Rank the remedies", "options": ["refund", "replace"]}, +) +_NATIVE_OPENAI_FIELDS: Final[Mapping[str, object]] = MappingProxyType( + {"prompt_cache_key": "refunds-v2", "prompt_cache_options": {"retention": "24h"}} +) +_NATIVE_OPENAI_RESPONSE: Final[Mapping[str, object]] = MappingProxyType( + { + "id": "decision_123", + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9, "rationale": "The screen is cracked."}, + {"type": "ranking", "name": "remedy", "ranking": ["refund", "replace"]}, + ], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + "cost_details": {"total_usd": 0.0001}, + }, + } +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("openai_default_base") +async def test_an_openai_deployment_behind_a_router_gets_the_caller_body_as_sent(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=dict(_NATIVE_OPENAI_RESPONSE)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_key": "deployment-key", + "organization": "org-deployment", + "drop_params": True, + "timeout": 30, + "max_retries": 0, + }, + "model_info": {"id": "luna-deployment"}, + } + ] + ) + + response: Final = await router.adecisions( + model="decider", + input=_NATIVE_OPENAI_INPUT, + questions=_NATIVE_OPENAI_QUESTIONS, + metadata={"trace": "caller"}, + **_NATIVE_OPENAI_FIELDS, + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "gpt-6-luna", + "input": list(_NATIVE_OPENAI_INPUT), + "questions": list(_NATIVE_OPENAI_QUESTIONS), + **_NATIVE_OPENAI_FIELDS, + } + assert response.model_dump(mode="json") == _NATIVE_OPENAI_RESPONSE + assert response._hidden_params["custom_llm_provider"] == "openai" + assert response._hidden_params["provider_response_model"] == "openai/gpt-6-luna" + + +@pytest.mark.parametrize( + "unsendable", + ({"prompt_cache_key": object()}, {"prompt_cache_options": {"tag": b"\xff\xfe"}}), + ids=("not_json", "not_utf8"), +) +@pytest.mark.usefixtures("openai_default_base") +def test_a_caller_value_that_cannot_be_sent_as_json_is_a_client_error_before_http( + unsendable: Mapping[str, object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=dict(_NATIVE_OPENAI_RESPONSE)) + + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request") as caught: + litellm.decisions( + model="openai/gpt-6-luna", api_key="caller-key", input="review", questions=_PREDICATE_QUESTIONS, **unsendable + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("is_async", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "upstream_body", + (json.dumps({**_NATIVE_OPENAI_RESPONSE, "answers": "not a list"}).encode(), b"busy", b"[]"), + ids=("malformed_answers", "not_json", "not_an_object"), +) +@pytest.mark.usefixtures("openai_default_base") +async def test_an_openai_response_litellm_cannot_parse_is_a_provider_error( + is_async: bool, upstream_body: bytes, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_OPENAI_DECISIONS_URL).respond(content=upstream_body, headers={"content-type": "application/json"}) + request: Final = { + "model": "openai/gpt-6-luna", + "api_key": "caller-key", + "input": "review", + "questions": _PREDICATE_QUESTIONS, + } + + with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"): + await litellm.adecisions(**request) if is_async else litellm.decisions(**request) + + +_NATIVE_UNSUPPORTED_REQUESTS: Final[tuple[tuple[str, Mapping[str, object], Sequence[Mapping[str, object]]], ...]] = ( + ( + "box questions", + MappingProxyType({"questions": (dict(_CRACK_BOX_QUESTION), *_PREDICATE_QUESTIONS)}), + ({"type": "refusal", "name": "crack"}, *_OPENAI_RESPONSE_ANSWERS), + ), + ( + "audio", + MappingProxyType({"questions": _PREDICATE_QUESTIONS, "audio": _AUDIO_CLIPS}), + _OPENAI_RESPONSE_ANSWERS, + ), +) +_NATIVE_UNSUPPORTED_IDS: Final = ("box", "audio") + + +@pytest.mark.usefixtures("openai_default_base") +@pytest.mark.parametrize( + ("param", "request_kwargs", "answers"), _NATIVE_UNSUPPORTED_REQUESTS, ids=_NATIVE_UNSUPPORTED_IDS +) +def test_an_openai_deployment_refuses_what_it_cannot_take_unless_it_drops_params( + param: str, + request_kwargs: Mapping[str, object], + answers: Sequence[Mapping[str, object]], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=_OPENAI_RESPONSE) + send: Final = partial( + litellm.decisions, + model="openai/gpt-6-luna", + api_key="caller-key", + input="review", + model_info={"supports_box_questions": False}, + **request_kwargs, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: + send() + assert caught.value.status_code == 400 + assert not route.called + + response: Final = send(drop_params=True) + + assert json.loads(route.calls.last.request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": list(_PREDICATE_QUESTIONS), + } + assert response.model_dump(mode="json")["answers"] == list(answers) + + @pytest.mark.asyncio @pytest.mark.parametrize("openai_format", (False, True)) @pytest.mark.parametrize( diff --git a/tests/unit/llms/azure/decisions/test_transformation.py b/tests/unit/llms/azure/decisions/test_transformation.py index b6de6084b9c..1b1d19d2d8c 100644 --- a/tests/unit/llms/azure/decisions/test_transformation.py +++ b/tests/unit/llms/azure/decisions/test_transformation.py @@ -95,6 +95,40 @@ async def test_azure_decisions_uses_entra_token_and_openai_wire_format( assert response._hidden_params["custom_llm_provider"] == "azure" +async def test_azure_decisions_send_the_caller_body_and_return_the_answers_as_sent( + respx_mock: respx.MockRouter, +) -> None: + sent_input: Final = [ + {"type": "message", "role": "developer", "content": "Judge exports only."}, + {"role": "user", "content": [{"type": "input_json", "json": {"job": "export", "state": "hung"}}]}, + ] + sent_questions: Final = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?", "rubric": "crashes only"} + ] + upstream_response: Final = {**_RESPONSE, "id": "decision_1", "answers": [{"type": "ranking", "name": "defect"}]} + route: Final = respx_mock.post(_URL).respond(json=upstream_response) + + response: Final = await litellm.adecisions( + model="azure/luna-deployment", + api_base=_BASE, + azure_ad_token="synthetic-entra", + input=sent_input, + questions=sent_questions, + prompt_cache_key="exports", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "luna-deployment", + "input": sent_input, + "questions": sent_questions, + "prompt_cache_key": "exports", + } + assert response.model_dump(mode="json", exclude={"usage"}) == { + key: value for key, value in upstream_response.items() if key != "usage" + } + assert response._hidden_params["custom_llm_provider"] == "azure" + + @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.parametrize("systemone", (False, True)) async def test_default_credential_refreshes_without_opt_in( diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 186ee545323..1faf6c68ff8 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -97,7 +97,7 @@ def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) -def test_an_openai_request_reaches_openai_unchanged() -> None: +def test_an_openai_request_round_trips_through_the_ir_unchanged() -> None: assert ir_to_openai_request("gpt-6-luna", _openai_ir(_OPENAI_REQUEST)) == { "model": "gpt-6-luna", **_OPENAI_REQUEST, diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..2e26f8fb92b 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -737,23 +737,7 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ response: Final = client.post("/v1/decisions", json=request_body) assert response.status_code == 200, response.text - assert json.loads(upstream.calls[0].request.content) == { - "model": "gpt-6-luna", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "The package arrived with a broken screen."}, - {"type": "input_text", "text": "I want a refund."}, - ], - }, - {"type": "message", "role": "user", "content": "Order 1234."}, - image_message, - ], - "questions": _OPENAI_FORMAT_REQUEST["questions"], - "safety_identifier": "end-user-1", - } + assert json.loads(upstream.calls[0].request.content) == {**request_body, "model": "gpt-6-luna"} body: Final = response.json() assert body["answers"] == _OPENAI_FORMAT_ANSWERS assert body["usage"] == upstream_usage @@ -765,6 +749,62 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "input": [ + {"type": "message", "role": "developer", "content": "Judge for the refunds team."}, + {"role": "user", "content": [{"type": "input_json", "json": {"order": 1234}}]}, + ], + "questions": [ + {"type": "choice", "instructions": "Pick one", "choices": [{"value": True}, {"value": "true"}]}, + ], + "prompt_cache_key": "refunds-v2", +} +_UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS: Final = [ + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 1.0}], + "confidence": 0.9, + } +] + + +@pytest.mark.parametrize("model", ("openai/gpt-6-luna", "perplexity/pplx-decider-v1-27b"), ids=("openai", "translated")) +def test_only_a_deployment_that_cannot_take_an_openai_format_body_as_sent_refuses_it( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + model: str, +) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + _route_decider_to(monkeypatch, {"model": model, "api_key": "k"}) + openai_upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 1, "total_tokens": _INPUT_TOKENS + 1}, + } + ) + perplexity_upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST) + + if model.startswith("perplexity/"): + assert response.status_code == 400, response.text + assert not perplexity_upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(openai_upstream.calls[0].request.content) == { + **_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST, + "model": "gpt-6-luna", + } + assert response.json()["answers"] == _UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS + + _CLEF_OMNI_MEDIA: Final[Mapping[str, object]] = { "audio": ["data:audio/wav;base64,YQ==", {"content_type": "audio/webm;codecs=opus", "base64": "Yg=="}], "videos": ["data:video/mp4;base64,Yw=="], diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 66fe9b17a82..e4b0fc34fce 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4764,14 +4764,15 @@ export interface paths { put?: never; /** * Ask questions about an input (OpenAI Decisions format) - * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * @description Judge an `input` against one or more `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - * any provider is called. `model` is any decision model in the proxy - * model list. System One decision models accept this format too, LiteLLM translates the request and the - * answers. Streaming is not supported. + * `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + * and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + * deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + * update. System One decision models accept this format too, LiteLLM translates the request and the answers and + * rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) */ @@ -20222,14 +20223,15 @@ export interface paths { put?: never; /** * Ask questions about an input (OpenAI Decisions format) - * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * @description Judge an `input` against one or more `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - * any provider is called. `model` is any decision model in the proxy - * model list. System One decision models accept this format too, LiteLLM translates the request and the - * answers. Streaming is not supported. + * `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + * and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + * deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + * update. System One decision models accept this format too, LiteLLM translates the request and the answers and + * rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) */ @@ -40354,10 +40356,13 @@ export interface components { /** Name */ name: string | null; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default box + * @constant */ type: "box"; + } & { + [key: string]: unknown; }; /** OpenAIChoiceAnswer */ OpenAIChoiceAnswer: { @@ -40370,10 +40375,13 @@ export interface components { /** Probabilities */ probabilities: components["schemas"]["OpenAIChoiceProbability"][]; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default choice + * @constant */ type: "choice"; + } & { + [key: string]: unknown; }; /** OpenAIChoiceProbability */ OpenAIChoiceProbability: { @@ -40381,6 +40389,8 @@ export interface components { probability: number; /** Value */ value: string | boolean; + } & { + [key: string]: unknown; }; /** OpenAIDecisionInputTokensDetails */ OpenAIDecisionInputTokensDetails: { @@ -40394,6 +40404,8 @@ export interface components { * @default 0 */ cached_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIDecisionOutputTokensDetails */ OpenAIDecisionOutputTokensDetails: { @@ -40402,11 +40414,13 @@ export interface components { * @default 0 */ reasoning_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIDecisionResponse */ OpenAIDecisionResponse: { /** Answers */ - answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIBoxAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIBoxAnswer"] | components["schemas"]["OpenAIRefusalAnswer"] | components["schemas"]["OpenAIUnknownAnswer"])[]; /** Model */ model: string; usage: components["schemas"]["OpenAIDecisionUsage"]; @@ -40434,6 +40448,8 @@ export interface components { output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; /** Total Tokens */ total_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIPredicateAnswer */ OpenAIPredicateAnswer: { @@ -40442,20 +40458,26 @@ export interface components { /** Probability */ probability: number; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default predicate + * @constant */ type: "predicate"; + } & { + [key: string]: unknown; }; /** OpenAIRefusalAnswer */ OpenAIRefusalAnswer: { /** Name */ name: string | null; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default refusal + * @constant */ type: "refusal"; + } & { + [key: string]: unknown; }; /** OpenAIScoreAnswer */ OpenAIScoreAnswer: { @@ -40468,10 +40490,13 @@ export interface components { /** Score */ score: number; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default score + * @constant */ type: "score"; + } & { + [key: string]: unknown; }; /** OpenAIScoreProbability */ OpenAIScoreProbability: { @@ -40481,6 +40506,8 @@ export interface components { probability: number; /** Value */ value: number; + } & { + [key: string]: unknown; }; /** OpenAIToolSearchToolParam */ OpenAIToolSearchToolParam: { @@ -40492,6 +40519,15 @@ export interface components { */ type: "tool_search"; }; + /** OpenAIUnknownAnswer */ + OpenAIUnknownAnswer: { + /** Name */ + name?: string | null; + /** Type */ + type: string; + } & { + [key: string]: unknown; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect.