From 61479093dd9ba6df1fb7871242da12fe8437fba9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:00:24 -0800 Subject: [PATCH] fix(decisions): forward OpenAI-format bodies to OpenAI and Azure deployments as sent An OpenAI or Azure OpenAI deployment on /v1/decisions now gets the caller's body unchanged, with model set to the deployment's model and LiteLLM settings removed, instead of a body rebuilt from the internal request shape. Fields LiteLLM does not model yet (developer messages, input_json parts, prompt cache breakpoints, array instructions, more than 128 questions) reach OpenAI, and OpenAI's answers come back as sent. Deployments of other providers keep translating and still answer 400 before calling the provider. --- litellm/decisions/call.py | 504 ++++++++++++++---- litellm/decisions/main.py | 8 +- .../llms/base_llm/decisions/transformation.py | 14 +- litellm/llms/custom_httpx/llm_http_handler.py | 82 ++- .../llms/openai/decisions/transformation.py | 15 +- litellm/proxy/_lazy_openapi_snapshot.json | 52 +- .../proxy/decisions_endpoints/endpoints.py | 30 +- litellm/systemone/main.py | 3 +- litellm/types/decisions.py | 68 ++- tests/unit/decisions/test_call.py | 38 ++ tests/unit/decisions/test_main.py | 205 ++++++- .../azure/decisions/test_transformation.py | 34 ++ .../test_openai_decisions_transformation.py | 2 +- .../decisions_endpoints/test_endpoints.py | 74 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 78 ++- 15 files changed, 998 insertions(+), 209 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 9c0e51e14ac..c44bc8505e4 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,7 +1,7 @@ import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace -from functools import partial +from functools import partial, reduce from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias @@ -35,7 +35,11 @@ from litellm.llms.base_llm.decisions.transformation import ( with_inlined_images, ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig, + ir_to_openai_response, + openai_request_to_ir, +) from litellm.types.decisions import ( DecisionQuestion, DecisionsAsset, @@ -56,15 +60,18 @@ from litellm.types.decisions import ( DecisionsRequestBody, DecisionsResponse, DecisionsVideo, + OpenAIDecisionAnswer, OpenAIDecisionInput, OpenAIDecisionQuestion, OpenAIDecisionRequestBody, OpenAIDecisionResponse, + OpenAIRefusalAnswer, UnsupportedDecisionsRequest, ) from litellm.types.llms.openai import ChatCompletionUserMessage +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode -from litellm.utils import ProviderConfigManager, get_model_cost_key +from litellm.utils import ProviderConfigManager, filter_out_litellm_params, get_model_cost_key SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] @@ -76,6 +83,9 @@ DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequest _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( + OpenAIDecisionQuestions | None +) _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( str | None, config=ConfigDict(title="safety_identifier") ) @@ -86,18 +96,19 @@ _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None] ] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None) _HANDLER: Final = BaseLLMHTTPHandler() +_LITELLM_SETTINGS: Final = frozenset((*GenericLiteLLMParams.model_fields, "base_url", "additional_drop_params", "user")) _BASE64_MARKER: Final = ";base64," _IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images")) -_IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) _ASSETS_ADAPTER: Final[TypeAdapter[DecisionsAssets | None]] = TypeAdapter( DecisionsAssets | None, config=ConfigDict(title="assets") ) _ASSET_ENTRY_ADAPTER: Final[TypeAdapter[tuple[DecisionsAssetReference, DecisionsAsset]]] = TypeAdapter( tuple[DecisionsAssetReference, DecisionsAsset] ) -_ASSET_ENTRIES_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) def _valid_image(image: object) -> DecisionsImage | None: @@ -107,9 +118,9 @@ def _valid_image(image: object) -> DecisionsImage | None: return None -def _image_entries(images: object) -> tuple[object, ...]: +def _entries(value: object) -> tuple[object, ...]: try: - return _IMAGE_ENTRIES_ADAPTER.validate_python(images) + return _ENTRIES_ADAPTER.validate_python(value) except ValidationError: return () @@ -129,9 +140,9 @@ def _valid_asset(entry: tuple[str, object]) -> tuple[str, DecisionsAsset] | None return None -def _asset_entries(assets: object) -> Mapping[str, object]: +def _fields(value: object) -> Mapping[str, object]: try: - return _ASSET_ENTRIES_ADAPTER.validate_python(assets) + return _FIELDS_ADAPTER.validate_python(value) except ValidationError: return MappingProxyType({}) @@ -140,7 +151,7 @@ def _request_assets(assets: object, drops_params: bool) -> Mapping[str, Decision if not drops_params: return _ASSETS_ADAPTER.validate_python(assets) valid: Final = MappingProxyType( - dict(entry for entry in map(_valid_asset, _asset_entries(assets).items()) if entry is not None) + dict(entry for entry in map(_valid_asset, _fields(assets).items()) if entry is not None) ) return valid or None @@ -148,7 +159,7 @@ def _request_assets(assets: object, drops_params: bool) -> Mapping[str, Decision def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]: if not drops_params: return _IMAGES_ADAPTER.validate_python(images) or () - return tuple(image for image in map(_valid_image, _image_entries(images)) if image is not None) + return tuple(image for image in map(_valid_image, _entries(images)) if image is not None) @dataclass(frozen=True, slots=True) @@ -176,7 +187,7 @@ class SystemOneRequestFields: @dataclass(frozen=True, slots=True) class OpenAIDecisionRequestFields: input: OpenAIDecisionInput | None - questions: OpenAIDecisionQuestions | None + questions: object audio: DecisionsAudioClips | None = None videos: DecisionsVideoClips | None = None @@ -212,6 +223,28 @@ class DecisionsCall: caller_questions: tuple[DecisionsIRQuestion, ...] +@dataclass(frozen=True, slots=True) +class DroppedQuestion: + position: int + name: str | None + + +@dataclass(frozen=True, slots=True, repr=False) +class NativeOpenAIDecisionsCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: OpenAIDecisionsConfig + body: Mapping[str, object] + dropped_questions: tuple[DroppedQuestion, ...] + api_base: str + api_key: str | None = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] = field(repr=False) + + def supported_decisions_providers() -> tuple[str, ...]: return tuple( provider.value @@ -505,18 +538,23 @@ def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> logging_obj.update_messages([_logged_request_message(caller_request)]) -def prepare_call( - *, +@dataclass(frozen=True, slots=True, repr=False) +class _Route: + requested_model: str + model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + api_base: str | None + api_key: str | None = field(repr=False) + + +def _route( model: str, - fields: RequestFields, - safety_identifier: str | None, - api_key: str | None, - api_base: str | None, - timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, - extra_headers: Mapping[str, str] | None, + api_base: str | None, + api_key: str | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> _Route: base_url_alias: Final = kwargs.get("base_url") call_api_base: Final = ( base_url_alias if api_base is None and isinstance(base_url_alias, str) and base_url_alias else api_base @@ -538,40 +576,118 @@ def prepare_call( canonical_model: Final = provider_config.canonical_model(upstream_model) except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error + return _Route( + requested_model=model, + model=canonical_model, + custom_llm_provider=provider, + provider_config=provider_config, + api_base=dynamic_api_base or call_api_base, + api_key=dynamic_api_key or api_key, + ) + + +def _invalid_request(error: ValueError | TypeError, route: _Route) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _connection(route: _Route, kwargs: Mapping[str, object]) -> tuple[str, str | None]: try: - headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs) - support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields) + resolved_api_base, static_api_key = route.provider_config.resolve_connection(route.api_base, route.api_key) + except (ValueError, httpx.InvalidURL) as error: + raise litellm.BadRequestError( + message=str(error), model=route.requested_model, llm_provider=route.custom_llm_provider + ) from error + if resolved_api_base is None: + raise litellm.BadRequestError( + message=route.provider_config.missing_api_base_message(route.custom_llm_provider), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + resolved_api_key: Final = route.provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs) + if resolved_api_key is None and route.provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{route.custom_llm_provider}'", + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + return resolved_api_base, resolved_api_key + + +def _unsupported_params_error(names: Sequence[str], route: _Route) -> litellm.UnsupportedParamsError: + return litellm.UnsupportedParamsError( + message=( + f"{route.custom_llm_provider} does not support parameters: {list(names)}, " + f"for model={route.requested_model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _nothing_left_error(nothing_left: _NothingLeftAfterDrop, route: _Route) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=( + f"{route.custom_llm_provider} does not support parameters: {list(nothing_left.names)}, " + f"for model={route.requested_model}, and dropping them leaves no {nothing_left.missing}" + ), + model=route.requested_model, + llm_provider=route.custom_llm_provider, + ) + + +def _call_logging_obj(route: _Route, api_base: str, kwargs: Mapping[str, object]) -> LiteLLMLoggingObj | None: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLoggingObj): + return None + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=route.model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": route.provider_config.get_complete_url(api_base, route.model), + "input_cost_per_query": ( + kwargs.get("input_cost_per_query") + if route.custom_llm_provider in QUERY_PRICED_DECISIONS_PROVIDERS + else None + ), + }, + custom_llm_provider=route.custom_llm_provider, + ) + return logging_obj + + +def _ir_call( + route: _Route, + fields: RequestFields, + safety_identifier: str | None, + timeout: float | httpx.Timeout | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + provider_config: Final = route.provider_config + checked_fields: Final = ( + replace(fields, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(fields.questions)) + if isinstance(fields, OpenAIDecisionRequestFields) + else fields + ) + try: + headers: Final = decisions_outbound_headers(extra_headers, route.custom_llm_provider, kwargs) + support: Final = _resolved_support( + provider_config, route.model, route.custom_llm_provider, headers, kwargs, checked_fields + ) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) - request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate( + request: Final = _fields_without_unsupported_media(checked_fields, support, kwargs).validate( safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs) ) except ValidationError as error: - raise litellm.BadRequestError( - message=f"Invalid Decisions request: {error}", - model=model, - llm_provider=provider, - ) from error - - try: - resolved_api_base, static_api_key = provider_config.resolve_connection( - dynamic_api_base or call_api_base, dynamic_api_key or api_key - ) - except (ValueError, httpx.InvalidURL) as error: - raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error - if resolved_api_base is None: - raise litellm.BadRequestError( - message=provider_config.missing_api_base_message(provider), - model=model, - llm_provider=provider, - ) - resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs) - if resolved_api_key is None and provider_config.api_key_required: - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, - ) - + raise _invalid_request(error, route) from error + resolved_api_base, resolved_api_key = _connection(route, kwargs) caller_ir_request: Final = _without_malformed_images( _ir_request(request, request_safety_identifier), _drops_params(kwargs) ) @@ -581,55 +697,26 @@ def prepare_call( f"Invalid Decisions request: {caller_ir_request.count} input_image part(s) have an image_url that is " "neither a data:image/...;base64, URL nor an http(s) URL" ), - model=model, - llm_provider=provider, + model=route.requested_model, + llm_provider=route.custom_llm_provider, ) ir_request: Final = _provider_ir_request( caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs ) if isinstance(ir_request, _UnsupportedParams): - raise litellm.UnsupportedParamsError( - message=( - f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}. " - "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" - ), - model=model, - llm_provider=provider, - ) + raise _unsupported_params_error(ir_request.names, route) if isinstance(ir_request, _NothingLeftAfterDrop): - raise litellm.BadRequestError( - message=( - f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, " - f"and dropping them leaves no {ir_request.missing}" - ), - model=model, - llm_provider=provider, - ) - logging_obj: Final = kwargs.get("litellm_logging_obj") - if isinstance(logging_obj, LiteLLMLoggingObj): - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=canonical_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), - "input_cost_per_query": ( - kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None - ), - }, - custom_llm_provider=provider, - ) + raise _nothing_left_error(ir_request, route) return DecisionsCall( - model=canonical_model, - requested_model=model, - custom_llm_provider=provider, + model=route.model, + requested_model=route.requested_model, + custom_llm_provider=route.custom_llm_provider, provider_config=provider_config, request=request, ir_request=ir_request, api_base=resolved_api_base, api_key=resolved_api_key, - logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + logging_obj=_call_logging_obj(route, resolved_api_base, kwargs), headers=headers, timeout=timeout, litellm_params=kwargs, @@ -637,6 +724,153 @@ def prepare_call( ) +@dataclass(frozen=True, slots=True) +class _NativeQuestions: + sent: object + dropped: tuple[DroppedQuestion, ...] + + +def _question_name(question: object) -> str | None: + name: Final = _fields(question).get("name") + return name if isinstance(name, str) else None + + +def _native_questions( + questions: object, support: _ResolvedSupport, fields: OpenAIDecisionRequestFields, kwargs: Mapping[str, object] +) -> _NativeQuestions | _UnsupportedParams | _NothingLeftAfterDrop: + entries: Final = _entries(questions) + box_positions: Final = frozenset( + position for position, question in enumerate(entries) if _fields(question).get("type") == "box" + ) + refuses_box_questions: Final = bool(box_positions) and not support.box_questions + unsupported: Final = ( + *(("box questions",) if refuses_box_questions else ()), + *(("audio",) if fields.audio and not support.audio else ()), + *(("videos",) if fields.videos and not support.videos else ()), + ) + if not unsupported: + return _NativeQuestions(sent=questions, dropped=()) + if not _drops_params(kwargs): + return _UnsupportedParams(names=unsupported) + if not refuses_box_questions: + return _NativeQuestions(sent=questions, dropped=()) + kept: Final = tuple(question for position, question in enumerate(entries) if position not in box_positions) + if not kept: + return _NothingLeftAfterDrop(names=unsupported, missing="questions to answer") + return _NativeQuestions( + sent=kept, + dropped=tuple( + DroppedQuestion(position=position, name=_question_name(entries[position])) + for position in sorted(box_positions) + ), + ) + + +def _plain_mapping(value: object) -> Mapping[str, object]: + if isinstance(value, Mapping): + return _FIELDS_ADAPTER.validate_python(value) + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def _sent_fields( + route: _Route, fields: OpenAIDecisionRequestFields, kwargs: Mapping[str, object] +) -> Mapping[str, object]: + explicit: Final = ( + ("input", fields.input), + ("questions", fields.questions), + ("audio", fields.audio), + ("videos", fields.videos), + ) + sent: Final = MappingProxyType( + { + **{name: value for name, value in explicit if value is not None}, + **filter_out_litellm_params(kwargs=kwargs, excluding=_LITELLM_SETTINGS), + } + ) + try: + return _FIELDS_ADAPTER.validate_python(to_jsonable_python(sent, exclude_none=True, fallback=_plain_mapping)) + except (TypeError, ValueError) as error: + raise _invalid_request(error, route) from error + + +def _native_body( + model: str, sent: Mapping[str, object], questions: object, safety_identifier: str | None, support: _ResolvedSupport +) -> Mapping[str, object]: + unsupported_media: Final = frozenset( + name for name, supported in (("audio", support.audio), ("videos", support.videos)) if not supported + ) + return MappingProxyType( + { + "model": model, + **{name: value for name, value in sent.items() if name not in unsupported_media}, + **({} if questions is None else {"questions": questions}), + **({} if safety_identifier is None else {"safety_identifier": safety_identifier}), + } + ) + + +def _native_openai_call( + route: _Route, + provider_config: OpenAIDecisionsConfig, + fields: OpenAIDecisionRequestFields, + safety_identifier: str | None, + timeout: float | httpx.Timeout | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> NativeOpenAIDecisionsCall: + try: + headers: Final = decisions_outbound_headers(extra_headers, route.custom_llm_provider, kwargs) + support: Final = _resolved_support( + provider_config, route.model, route.custom_llm_provider, headers, kwargs, fields + ) + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) + except ValidationError as error: + raise _invalid_request(error, route) from error + resolved_api_base, resolved_api_key = _connection(route, kwargs) + sent: Final = _sent_fields(route, fields, kwargs) + questions: Final = _native_questions(sent.get("questions"), support, fields, kwargs) + if isinstance(questions, _UnsupportedParams): + raise _unsupported_params_error(questions.names, route) + if isinstance(questions, _NothingLeftAfterDrop): + raise _nothing_left_error(questions, route) + return NativeOpenAIDecisionsCall( + model=route.model, + requested_model=route.requested_model, + custom_llm_provider=route.custom_llm_provider, + provider_config=provider_config, + body=_native_body( + provider_config.request_model(route.model), sent, questions.sent, request_safety_identifier, support + ), + dropped_questions=questions.dropped, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=_call_logging_obj(route, resolved_api_base, kwargs), + headers=headers, + timeout=timeout, + litellm_params=kwargs, + ) + + +def prepare_call( + *, + model: str, + fields: RequestFields, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall | NativeOpenAIDecisionsCall: + route: Final = _route(model, custom_llm_provider, api_base, api_key, kwargs) + if isinstance(fields, OpenAIDecisionRequestFields) and isinstance(route.provider_config, OpenAIDecisionsConfig): + return _native_openai_call( + route, route.provider_config, fields, safety_identifier, timeout, extra_headers, kwargs + ) + return _ir_call(route, fields, safety_identifier, timeout, extra_headers, kwargs) + + def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]: if not call.provider_config.inlines_remote_images: return () @@ -753,8 +987,8 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: return body -def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str: - returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") +def _pricing_model(returned_model: str | None, call: DecisionsCall | NativeOpenAIDecisionsCall) -> str: + returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{returned_model}") if returned is not None and cost_map_key_prices_anything(returned): return returned return f"{call.custom_llm_provider}/{call.model}" @@ -767,7 +1001,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci { "model": f"{call.custom_llm_provider}/{call.model}", "custom_llm_provider": call.custom_llm_provider, - "provider_response_model": _pricing_model(response, call), + "provider_response_model": _pricing_model(response.model, call), **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), } ) @@ -798,7 +1032,7 @@ def _formatted_response( assert_never(call.request) -def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: +def _map_upstream_exception(error: Exception, call: DecisionsCall | NativeOpenAIDecisionsCall) -> Exception: provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" model: Final = f"{call.custom_llm_provider}/{call.model}" match error: @@ -816,7 +1050,7 @@ def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: ) -async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: +async def _asend_ir(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported))) body: Final = _request_body(inlined) @@ -840,7 +1074,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon return _format_response(response, inlined) -def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: +def _send_ir(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: supported: Final = _checked_image_count(_checked_image_types(call)) inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported))) body: Final = _request_body(inlined) @@ -864,6 +1098,90 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: return _format_response(response, inlined) +def _with_refusal( + answers: tuple[OpenAIDecisionAnswer, ...], dropped: DroppedQuestion +) -> tuple[OpenAIDecisionAnswer, ...]: + return (*answers[: dropped.position], OpenAIRefusalAnswer(name=dropped.name), *answers[dropped.position :]) + + +def _format_native_response( + response: OpenAIDecisionResponse, call: NativeOpenAIDecisionsCall +) -> OpenAIDecisionResponse: + formatted: Final = response.model_copy( + update={"answers": reduce(_with_refusal, call.dropped_questions, response.answers)} + ) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": _pricing_model(response.model, call), + } + ) + return formatted + + +async def _asend_native(call: NativeOpenAIDecisionsCall) -> OpenAIDecisionResponse: + try: + raw_response: Final = await _HANDLER.apost_decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + logged_body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + response: Final = call.provider_config.parse_native_response(call.custom_llm_provider, raw_response) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_native_response(response, call) + + +def _send_native(call: NativeOpenAIDecisionsCall) -> OpenAIDecisionResponse: + try: + raw_response: Final = _HANDLER.post_decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + logged_body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + response: Final = call.provider_config.parse_native_response(call.custom_llm_provider, raw_response) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_native_response(response, call) + + +async def asend(call: DecisionsCall | NativeOpenAIDecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + match call: + case NativeOpenAIDecisionsCall(): + return await _asend_native(call) + case DecisionsCall(): + return await _asend_ir(call) + case _: + assert_never(call) + + +def send(call: DecisionsCall | NativeOpenAIDecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + match call: + case NativeOpenAIDecisionsCall(): + return _send_native(call) + case DecisionsCall(): + return _send_ir(call) + case _: + assert_never(call) + + def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None: if field_name not in kwargs: return diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 4e609fd2ab8..c39f1e294fb 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -11,6 +11,7 @@ from litellm.decisions.call import ( DecisionsAudioClips, DecisionsCall, DecisionsVideoClips, + NativeOpenAIDecisionsCall, OpenAIDecisionQuestions, OpenAIDecisionRequestFields, RequestFields, @@ -27,9 +28,6 @@ from litellm.utils import client _STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None) _SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None) -_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( - OpenAIDecisionQuestions | None -) _SYSTEMONE_FIELDS: Final = frozenset({"state", "images", "assets"}) @@ -45,7 +43,7 @@ def _request_fields( reject_images_next_to_input(model=model, kwargs=kwargs) return OpenAIDecisionRequestFields( input=decision_input, - questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions), + questions=questions, audio=audio, videos=videos, ) @@ -78,7 +76,7 @@ def _prepare_decisions_call( custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> DecisionsCall | NativeOpenAIDecisionsCall: log_caller_request( kwargs, { diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 254783edf39..2db8a4ff0da 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -431,6 +431,14 @@ class DecisionsResponseValidationError(BaseLLMException): pass +def unexpected_response_error(custom_llm_provider: str, error: ValidationError) -> DecisionsResponseValidationError: + return DecisionsResponseValidationError( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " + f"{error.title} {_validation_summary(error)}", + ) + + def _lacks_confidence(answer: DecisionsIRAnswer) -> bool: return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None @@ -690,11 +698,7 @@ class BaseDecisionsConfig(ABC): payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) response: Final = self.parse_response(payload, request) except ValidationError as error: - raise DecisionsResponseValidationError( - status_code=500, - message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " - f"{error.title} {_validation_summary(error)}", - ) from error + raise unexpected_response_error(custom_llm_provider, error) from error if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers): raise DecisionsResponseValidationError( status_code=500, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2f17dc6dd10..7b72bdfecb7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1751,13 +1751,12 @@ class BaseLLMHTTPHandler: ) return url, outbound_headers, data - def decisions( + def post_decisions( self, model: str, custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsIRRequest, body: Mapping[str, object], logged_body: Mapping[str, object], api_base: str, @@ -1766,7 +1765,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None, litellm_params: Mapping[str, object], client: HTTPHandler | None = None, - ) -> DecisionsIRResponse: + ) -> httpx.Response: url, outbound_headers, data = self._prepare_decisions_request( model=model, logging_obj=logging_obj, @@ -1788,7 +1787,7 @@ class BaseLLMHTTPHandler: ) sync_httpx_client: Final = client if client is not None else get_httpx_client() try: - response: Final = sync_httpx_client.post( + return sync_httpx_client.post( url, content=signed_body, json=dict(data) if signed_body is None else None, @@ -1798,17 +1797,13 @@ class BaseLLMHTTPHandler: ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) - return provider_config.transform_decisions_response( - model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request - ) - async def adecisions( + async def apost_decisions( self, model: str, custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsIRRequest, body: Mapping[str, object], logged_body: Mapping[str, object], api_base: str, @@ -1817,7 +1812,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None, litellm_params: Mapping[str, object], client: AsyncHTTPHandler | None = None, - ) -> DecisionsIRResponse: + ) -> httpx.Response: url, outbound_headers, data = await self._aprepare_decisions_request( model=model, logging_obj=logging_obj, @@ -1847,7 +1842,7 @@ class BaseLLMHTTPHandler: else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) ) try: - response: Final = await async_httpx_client.post( + return await async_httpx_client.post( url, content=signed_body, json=dict(data) if signed_body is None else None, @@ -1857,6 +1852,71 @@ class BaseLLMHTTPHandler: ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + logged_body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: HTTPHandler | None = None, + ) -> DecisionsIRResponse: + response: Final = self.post_decisions( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + logged_body=logged_body, + api_base=api_base, + api_key=api_key, + headers=headers, + timeout=timeout, + litellm_params=litellm_params, + client=client, + ) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + logged_body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> DecisionsIRResponse: + response: Final = await self.apost_decisions( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + logged_body=logged_body, + api_base=api_base, + api_key=api_key, + headers=headers, + timeout=timeout, + litellm_params=litellm_params, + client=client, + ) return provider_config.transform_decisions_response( model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 5aa72579d6a..b2587efd0a7 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,11 +1,16 @@ from collections.abc import Mapping, Sequence from typing import Final -from pydantic import TypeAdapter +import httpx +from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never import litellm -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + decisions_text, + unexpected_response_error, +) from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( @@ -374,3 +379,9 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig): def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) + + def parse_native_response(self, custom_llm_provider: str, raw_response: httpx.Response) -> OpenAIDecisionResponse: + try: + return _OPENAI_RESPONSE_ADAPTER.validate_json(raw_response.content) + except ValidationError as error: + raise unexpected_response_error(custom_llm_provider, error) from error diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5cc1c53d396..db68e324cf7 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10638,6 +10638,7 @@ "type": "object" }, "OpenAIBoxAnswer": { + "additionalProperties": true, "properties": { "box": { "$ref": "#/components/schemas/DecisionsBox" @@ -10680,6 +10681,7 @@ "type": "object" }, "OpenAIChoiceAnswer": { + "additionalProperties": true, "properties": { "choice": { "anyOf": [ @@ -10737,6 +10739,7 @@ "type": "object" }, "OpenAIChoiceProbability": { + "additionalProperties": true, "properties": { "probability": { "title": "Probability", @@ -10762,6 +10765,7 @@ "type": "object" }, "OpenAIDecisionInputTokensDetails": { + "additionalProperties": true, "properties": { "cache_write_tokens": { "default": 0, @@ -10778,6 +10782,7 @@ "type": "object" }, "OpenAIDecisionOutputTokensDetails": { + "additionalProperties": true, "properties": { "reasoning_tokens": { "default": 0, @@ -10793,16 +10798,6 @@ "properties": { "answers": { "items": { - "discriminator": { - "mapping": { - "box": "#/components/schemas/OpenAIBoxAnswer", - "choice": "#/components/schemas/OpenAIChoiceAnswer", - "predicate": "#/components/schemas/OpenAIPredicateAnswer", - "refusal": "#/components/schemas/OpenAIRefusalAnswer", - "score": "#/components/schemas/OpenAIScoreAnswer" - }, - "propertyName": "type" - }, "oneOf": [ { "$ref": "#/components/schemas/OpenAIPredicateAnswer" @@ -10818,6 +10813,9 @@ }, { "$ref": "#/components/schemas/OpenAIRefusalAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIUnknownAnswer" } ] }, @@ -10841,6 +10839,7 @@ "type": "object" }, "OpenAIDecisionUsage": { + "additionalProperties": true, "properties": { "input_tokens": { "title": "Input Tokens", @@ -10877,6 +10876,7 @@ "type": "object" }, "OpenAIPredicateAnswer": { + "additionalProperties": true, "properties": { "name": { "anyOf": [ @@ -10908,6 +10908,7 @@ "type": "object" }, "OpenAIRefusalAnswer": { + "additionalProperties": true, "properties": { "name": { "anyOf": [ @@ -10934,6 +10935,7 @@ "type": "object" }, "OpenAIScoreAnswer": { + "additionalProperties": true, "properties": { "confidence": { "anyOf": [ @@ -10984,6 +10986,7 @@ "type": "object" }, "OpenAIScoreProbability": { + "additionalProperties": true, "properties": { "label": { "title": "Label", @@ -11006,6 +11009,31 @@ "title": "OpenAIScoreProbability", "type": "object" }, + "OpenAIUnknownAnswer": { + "additionalProperties": true, + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "OpenAIUnknownAnswer", + "type": "object" + }, "ScoreAnswer": { "additionalProperties": true, "properties": { @@ -11070,7 +11098,7 @@ "paths": { "/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against one or more `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI\nand Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the\ndeployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM\nupdate. System One decision models accept this format too, LiteLLM translates the request and the answers and\nrejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", "requestBody": { "content": { @@ -12476,7 +12504,7 @@ }, "/v1/decisions": { "post": { - "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before\nany provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge an `input` against one or more `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the\n`instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI\nand Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the\ndeployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM\nupdate. System One decision models accept this format too, LiteLLM translates the request and the answers and\nrejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", "requestBody": { "content": { diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index a8684dd9757..1f34a9e8c64 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -4,7 +4,7 @@ from typing import Annotated, Final, Literal, TypeAlias from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth @@ -15,7 +15,6 @@ from litellm.types.decisions import ( DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequest, - OpenAIDecisionRequestBody, OpenAIDecisionResponse, ) from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, ExtractionResponse @@ -23,9 +22,15 @@ from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, E router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) -_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( - OpenAIDecisionRequestBody -) + + +class _OpenAIFormatBody(BaseModel): + state: None = None + + model_config = ConfigDict(extra="allow", frozen=True, title="OpenAI format Decisions request") + + +_OPENAI_FORMAT_BODY_ADAPTER: Final[TypeAdapter[_OpenAIFormatBody]] = TypeAdapter(_OpenAIFormatBody) _EXTRACTION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) @@ -139,7 +144,7 @@ async def _process_decisions( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, body_adapter: ( - TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody] | TypeAdapter[ExtractionRequestBody] + TypeAdapter[DecisionsRequestBody] | TypeAdapter[_OpenAIFormatBody] | TypeAdapter[ExtractionRequestBody] ), route_type: _RouteType, ) -> object: @@ -272,14 +277,15 @@ async def decisions( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ - Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + Judge an `input` against one or more `questions` and get the answers back in the same order. Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - any provider is called. `model` is any decision model in the proxy - model list. System One decision models accept this format too, LiteLLM translates the request and the - answers. Streaming is not supported. + `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + update. System One decision models accept this format too, LiteLLM translates the request and the answers and + rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. [Docs](https://docs.litellm.ai/docs/decisions) """ @@ -287,7 +293,7 @@ async def decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, + body_adapter=_OPENAI_FORMAT_BODY_ADAPTER, route_type="adecisions", ) diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 27fa84e3f9b..cb965676487 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -11,6 +11,7 @@ from litellm.decisions.call import ( DecisionsCall, DecisionsImages, DecisionsVideoClips, + NativeOpenAIDecisionsCall, SystemOneQuestions, SystemOneRequestFields, asend, @@ -39,7 +40,7 @@ def _prepare_systemone_call( custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], -) -> DecisionsCall: +) -> DecisionsCall | NativeOpenAIDecisionsCall: fields: Final = SystemOneRequestFields( state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 3b28225f0ff..5eb535369a1 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -3,7 +3,18 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias -from pydantic import AfterValidator, ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config +from pydantic import ( + AfterValidator, + ConfigDict, + Discriminator, + Field, + PrivateAttr, + Tag, + TypeAdapter, + field_validator, + model_validator, + with_config, +) from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -416,14 +427,14 @@ class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): name: str | None probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIChoiceProbability(LiteLLMPydanticObjectBase): value: str | bool probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): @@ -433,7 +444,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): probabilities: tuple[OpenAIChoiceProbability, ...] confidence: float | None = None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIScoreProbability(LiteLLMPydanticObjectBase): @@ -441,7 +452,7 @@ class OpenAIScoreProbability(LiteLLMPydanticObjectBase): label: str probability: float - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): @@ -451,14 +462,14 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): probabilities: tuple[OpenAIScoreProbability, ...] confidence: float | None = None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): type: Literal["refusal"] = "refusal" name: str | None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIBoxAnswer(LiteLLMPydanticObjectBase): @@ -467,12 +478,43 @@ class OpenAIBoxAnswer(LiteLLMPydanticObjectBase): box: DecisionsBox confidence: float | None - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) + + +class OpenAIUnknownAnswer(LiteLLMPydanticObjectBase): + type: str + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +_KNOWN_OPENAI_ANSWER_TYPES: Final = frozenset({"predicate", "choice", "score", "box", "refusal"}) + + +class _TaggedAnswer(TypedDict, total=False): + type: ReadOnly[object] + + +_TAGGED_ANSWER_ADAPTER: Final[TypeAdapter[_TaggedAnswer]] = TypeAdapter(_TaggedAnswer) + + +def _openai_answer_tag(answer: object) -> str: + answer_type: Final = ( + _TAGGED_ANSWER_ADAPTER.validate_python(answer).get("type") + if isinstance(answer, Mapping) + else getattr(answer, "type", None) + ) + return answer_type if isinstance(answer_type, str) and answer_type in _KNOWN_OPENAI_ANSWER_TYPES else "unknown" OpenAIDecisionAnswer: TypeAlias = Annotated[ - OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIBoxAnswer | OpenAIRefusalAnswer, - Field(discriminator="type"), + Annotated[OpenAIPredicateAnswer, Tag("predicate")] + | Annotated[OpenAIChoiceAnswer, Tag("choice")] + | Annotated[OpenAIScoreAnswer, Tag("score")] + | Annotated[OpenAIBoxAnswer, Tag("box")] + | Annotated[OpenAIRefusalAnswer, Tag("refusal")] + | Annotated[OpenAIUnknownAnswer, Tag("unknown")], + Discriminator(_openai_answer_tag), ] @@ -480,13 +522,13 @@ class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): cached_tokens: int = 0 cache_write_tokens: int = 0 - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionOutputTokensDetails(LiteLLMPydanticObjectBase): reasoning_tokens: int = 0 - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): @@ -496,7 +538,7 @@ class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): output_tokens_details: OpenAIDecisionOutputTokensDetails = OpenAIDecisionOutputTokensDetails() total_tokens: int - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) class OpenAIDecisionResponse(_HiddenParamsResponse): diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py index 36a16dd5ad1..3b8db808208 100644 --- a/tests/unit/decisions/test_call.py +++ b/tests/unit/decisions/test_call.py @@ -7,6 +7,7 @@ from typing import Final import pytest import litellm +from litellm.decisions.call import NativeOpenAIDecisionsCall from litellm.decisions.main import _prepare_decisions_call from litellm.litellm_core_utils.litellm_logging import Logging from litellm.systemone.main import _prepare_systemone_call @@ -61,6 +62,43 @@ def test_systemone_logs_every_request_field_the_caller_sent(): assert set(logged) == {"state", "questions"} +def test_an_openai_body_carries_the_caller_fields_and_none_of_the_settings_litellm_owns( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + question: Final = {"type": "predicate", "instructions": QUESTION_TEXT, "rubric": "strict"} + + call: Final = _prepare_decisions_call( + **{**CONNECTION, "model": "openai/gpt-6-luna", "api_base": None}, + decision_input="My card was charged twice.", + questions=[question], + audio=None, + videos=None, + kwargs={ + "prompt_cache_key": "billing-v1", + "user": "key-hash", + "drop_params": True, + "organization": "org-1", + "base_url": "https://elsewhere.example/v1", + "additional_drop_params": ["temperature"], + "aws_region_name": "us-east-1", + "vertex_project": "project-1", + "azure_ad_token": "token-1", + "metadata": {"tags": ["billing"]}, + }, + ) + + assert isinstance(call, NativeOpenAIDecisionsCall) + assert call.body == { + "model": "gpt-6-luna", + "input": "My card was charged twice.", + "questions": [question], + "prompt_cache_key": "billing-v1", + } + + def test_decisions_logs_every_request_field_the_caller_sent(): logging_obj: Final = _logging_obj("adecisions") diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 789d59bd613..f459c863300 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -134,12 +134,13 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ) +_OPENAI_RESPONSE_ANSWERS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "sentiment"}, +) _OPENAI_RESPONSE: Final[Mapping[str, object]] = { "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "is_defect", "probability": 0.9}, - {"type": "refusal", "name": "sentiment"}, - ], + "answers": list(_OPENAI_RESPONSE_ANSWERS), "usage": { "input_tokens": _INPUT_TOKENS, "input_tokens_details": {"cached_tokens": _CACHED_TOKENS, "cache_write_tokens": _CACHE_WRITE_TOKENS}, @@ -2364,22 +2365,20 @@ def openai_default_base(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("OPENAI_API_BASE", raising=False) -@pytest.mark.usefixtures("cloudflare_account", "openai_default_base") -@pytest.mark.parametrize( - ("model", "upstream"), - (("cloudflare/clef", f"{_CLOUDFLARE_RUN_URL}/clef"), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)), - ids=("clef", "luna"), -) +@pytest.mark.usefixtures("cloudflare_account") @pytest.mark.parametrize("image_url", _MALFORMED_IMAGE_URLS, ids=_MALFORMED_IMAGE_URL_IDS) def test_malformed_input_image_urls_are_rejected_before_http( - model: str, upstream: str, image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + image_url: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - route: Final = respx_mock.post(upstream).respond(json=_RESPONSE) + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match=r"Invalid Decisions request: 1 input_image") as caught: litellm.decisions( - model=model, api_key="caller-key", input=_screen_messages(image_url), questions=_PREDICATE_QUESTIONS + model="cloudflare/clef", + api_key="caller-key", + input=_screen_messages(image_url), + questions=_PREDICATE_QUESTIONS, ) assert caught.value.status_code == 400 @@ -4085,22 +4084,196 @@ async def test_an_openai_format_deployment_that_advertises_media_is_still_refuse assert not route.called -def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None: - route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) +@pytest.mark.usefixtures("openai_default_base") +@pytest.mark.parametrize( + ("model", "upstream"), + (("perplexity/pplx-decider-v1-27b", _PERPLEXITY_DECISIONS_URL), ("openai/gpt-6-luna", _OPENAI_DECISIONS_URL)), + ids=("translated", "openai_native"), +) +def test_dropping_every_question_answers_400_before_http( + model: str, upstream: str, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(upstream).respond(json=_RESPONSE) with pytest.raises(litellm.BadRequestError, match="leaves no questions to answer") as caught: litellm.decisions( - model="perplexity/pplx-decider-v1-27b", + model=model, input="review", questions=[{"type": "box", "name": "crack", "instructions": "the crack on the screen"}], api_key="caller-key", drop_params=True, + model_info={"supports_box_questions": False}, ) assert caught.value.status_code == 400 assert not route.called +_NATIVE_OPENAI_INPUT: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "message", "role": "developer", "content": "Judge for the refunds team."}, + {"role": "user", "content": [{"type": "input_json", "json": {"order": 1234, "status": "damaged"}}]}, +) +_NATIVE_OPENAI_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?", "rubric": "visible damage only"}, + {"type": "ranking", "name": "remedy", "instructions": "Rank the remedies", "options": ["refund", "replace"]}, +) +_NATIVE_OPENAI_FIELDS: Final[Mapping[str, object]] = MappingProxyType( + {"prompt_cache_key": "refunds-v2", "prompt_cache_options": {"retention": "24h"}} +) +_NATIVE_OPENAI_RESPONSE: Final[Mapping[str, object]] = MappingProxyType( + { + "id": "decision_123", + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9, "rationale": "The screen is cracked."}, + {"type": "ranking", "name": "remedy", "ranking": ["refund", "replace"]}, + ], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + "cost_details": {"total_usd": 0.0001}, + }, + } +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("openai_default_base") +async def test_an_openai_deployment_behind_a_router_gets_the_caller_body_as_sent(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=dict(_NATIVE_OPENAI_RESPONSE)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_key": "deployment-key", + "organization": "org-deployment", + "drop_params": True, + "timeout": 30, + "max_retries": 0, + }, + "model_info": {"id": "luna-deployment"}, + } + ] + ) + + response: Final = await router.adecisions( + model="decider", + input=_NATIVE_OPENAI_INPUT, + questions=_NATIVE_OPENAI_QUESTIONS, + metadata={"trace": "caller"}, + **_NATIVE_OPENAI_FIELDS, + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "gpt-6-luna", + "input": list(_NATIVE_OPENAI_INPUT), + "questions": list(_NATIVE_OPENAI_QUESTIONS), + **_NATIVE_OPENAI_FIELDS, + } + assert response.model_dump(mode="json") == _NATIVE_OPENAI_RESPONSE + assert response._hidden_params["custom_llm_provider"] == "openai" + assert response._hidden_params["provider_response_model"] == "openai/gpt-6-luna" + + +@pytest.mark.parametrize( + "unsendable", + ({"prompt_cache_key": object()}, {"prompt_cache_options": {"tag": b"\xff\xfe"}}), + ids=("not_json", "not_utf8"), +) +@pytest.mark.usefixtures("openai_default_base") +def test_a_caller_value_that_cannot_be_sent_as_json_is_a_client_error_before_http( + unsendable: Mapping[str, object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=dict(_NATIVE_OPENAI_RESPONSE)) + + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request") as caught: + litellm.decisions( + model="openai/gpt-6-luna", api_key="caller-key", input="review", questions=_PREDICATE_QUESTIONS, **unsendable + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("is_async", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "upstream_body", + (json.dumps({**_NATIVE_OPENAI_RESPONSE, "answers": "not a list"}).encode(), b"busy", b"[]"), + ids=("malformed_answers", "not_json", "not_an_object"), +) +@pytest.mark.usefixtures("openai_default_base") +async def test_an_openai_response_litellm_cannot_parse_is_a_provider_error( + is_async: bool, upstream_body: bytes, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_OPENAI_DECISIONS_URL).respond(content=upstream_body, headers={"content-type": "application/json"}) + request: Final = { + "model": "openai/gpt-6-luna", + "api_key": "caller-key", + "input": "review", + "questions": _PREDICATE_QUESTIONS, + } + + with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"): + await litellm.adecisions(**request) if is_async else litellm.decisions(**request) + + +_NATIVE_UNSUPPORTED_REQUESTS: Final[tuple[tuple[str, Mapping[str, object], Sequence[Mapping[str, object]]], ...]] = ( + ( + "box questions", + MappingProxyType({"questions": (dict(_CRACK_BOX_QUESTION), *_PREDICATE_QUESTIONS)}), + ({"type": "refusal", "name": "crack"}, *_OPENAI_RESPONSE_ANSWERS), + ), + ( + "audio", + MappingProxyType({"questions": _PREDICATE_QUESTIONS, "audio": _AUDIO_CLIPS}), + _OPENAI_RESPONSE_ANSWERS, + ), +) +_NATIVE_UNSUPPORTED_IDS: Final = ("box", "audio") + + +@pytest.mark.usefixtures("openai_default_base") +@pytest.mark.parametrize( + ("param", "request_kwargs", "answers"), _NATIVE_UNSUPPORTED_REQUESTS, ids=_NATIVE_UNSUPPORTED_IDS +) +def test_an_openai_deployment_refuses_what_it_cannot_take_unless_it_drops_params( + param: str, + request_kwargs: Mapping[str, object], + answers: Sequence[Mapping[str, object]], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_OPENAI_DECISIONS_URL).respond(json=_OPENAI_RESPONSE) + send: Final = partial( + litellm.decisions, + model="openai/gpt-6-luna", + api_key="caller-key", + input="review", + model_info={"supports_box_questions": False}, + **request_kwargs, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\['{param}'\].*drop_params") as caught: + send() + assert caught.value.status_code == 400 + assert not route.called + + response: Final = send(drop_params=True) + + assert json.loads(route.calls.last.request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": list(_PREDICATE_QUESTIONS), + } + assert response.model_dump(mode="json")["answers"] == list(answers) + + @pytest.mark.asyncio @pytest.mark.parametrize("openai_format", (False, True)) @pytest.mark.parametrize( diff --git a/tests/unit/llms/azure/decisions/test_transformation.py b/tests/unit/llms/azure/decisions/test_transformation.py index b6de6084b9c..1b1d19d2d8c 100644 --- a/tests/unit/llms/azure/decisions/test_transformation.py +++ b/tests/unit/llms/azure/decisions/test_transformation.py @@ -95,6 +95,40 @@ async def test_azure_decisions_uses_entra_token_and_openai_wire_format( assert response._hidden_params["custom_llm_provider"] == "azure" +async def test_azure_decisions_send_the_caller_body_and_return_the_answers_as_sent( + respx_mock: respx.MockRouter, +) -> None: + sent_input: Final = [ + {"type": "message", "role": "developer", "content": "Judge exports only."}, + {"role": "user", "content": [{"type": "input_json", "json": {"job": "export", "state": "hung"}}]}, + ] + sent_questions: Final = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?", "rubric": "crashes only"} + ] + upstream_response: Final = {**_RESPONSE, "id": "decision_1", "answers": [{"type": "ranking", "name": "defect"}]} + route: Final = respx_mock.post(_URL).respond(json=upstream_response) + + response: Final = await litellm.adecisions( + model="azure/luna-deployment", + api_base=_BASE, + azure_ad_token="synthetic-entra", + input=sent_input, + questions=sent_questions, + prompt_cache_key="exports", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "luna-deployment", + "input": sent_input, + "questions": sent_questions, + "prompt_cache_key": "exports", + } + assert response.model_dump(mode="json", exclude={"usage"}) == { + key: value for key, value in upstream_response.items() if key != "usage" + } + assert response._hidden_params["custom_llm_provider"] == "azure" + + @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.parametrize("systemone", (False, True)) async def test_default_credential_refreshes_without_opt_in( diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 186ee545323..1faf6c68ff8 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -97,7 +97,7 @@ def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) -def test_an_openai_request_reaches_openai_unchanged() -> None: +def test_an_openai_request_round_trips_through_the_ir_unchanged() -> None: assert ir_to_openai_request("gpt-6-luna", _openai_ir(_OPENAI_REQUEST)) == { "model": "gpt-6-luna", **_OPENAI_REQUEST, diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..2e26f8fb92b 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -737,23 +737,7 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ response: Final = client.post("/v1/decisions", json=request_body) assert response.status_code == 200, response.text - assert json.loads(upstream.calls[0].request.content) == { - "model": "gpt-6-luna", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "The package arrived with a broken screen."}, - {"type": "input_text", "text": "I want a refund."}, - ], - }, - {"type": "message", "role": "user", "content": "Order 1234."}, - image_message, - ], - "questions": _OPENAI_FORMAT_REQUEST["questions"], - "safety_identifier": "end-user-1", - } + assert json.loads(upstream.calls[0].request.content) == {**request_body, "model": "gpt-6-luna"} body: Final = response.json() assert body["answers"] == _OPENAI_FORMAT_ANSWERS assert body["usage"] == upstream_usage @@ -765,6 +749,62 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "input": [ + {"type": "message", "role": "developer", "content": "Judge for the refunds team."}, + {"role": "user", "content": [{"type": "input_json", "json": {"order": 1234}}]}, + ], + "questions": [ + {"type": "choice", "instructions": "Pick one", "choices": [{"value": True}, {"value": "true"}]}, + ], + "prompt_cache_key": "refunds-v2", +} +_UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS: Final = [ + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 1.0}], + "confidence": 0.9, + } +] + + +@pytest.mark.parametrize("model", ("openai/gpt-6-luna", "perplexity/pplx-decider-v1-27b"), ids=("openai", "translated")) +def test_only_a_deployment_that_cannot_take_an_openai_format_body_as_sent_refuses_it( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + model: str, +) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + _route_decider_to(monkeypatch, {"model": model, "api_key": "k"}) + openai_upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 1, "total_tokens": _INPUT_TOKENS + 1}, + } + ) + perplexity_upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST) + + if model.startswith("perplexity/"): + assert response.status_code == 400, response.text + assert not perplexity_upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(openai_upstream.calls[0].request.content) == { + **_UNTRANSLATABLE_OPENAI_FORMAT_REQUEST, + "model": "gpt-6-luna", + } + assert response.json()["answers"] == _UNTRANSLATABLE_OPENAI_FORMAT_ANSWERS + + _CLEF_OMNI_MEDIA: Final[Mapping[str, object]] = { "audio": ["data:audio/wav;base64,YQ==", {"content_type": "audio/webm;codecs=opus", "base64": "Yg=="}], "videos": ["data:video/mp4;base64,Yw=="], diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 66fe9b17a82..e4b0fc34fce 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4764,14 +4764,15 @@ export interface paths { put?: never; /** * Ask questions about an input (OpenAI Decisions format) - * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * @description Judge an `input` against one or more `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - * any provider is called. `model` is any decision model in the proxy - * model list. System One decision models accept this format too, LiteLLM translates the request and the - * answers. Streaming is not supported. + * `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + * and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + * deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + * update. System One decision models accept this format too, LiteLLM translates the request and the answers and + * rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) */ @@ -20222,14 +20223,15 @@ export interface paths { put?: never; /** * Ask questions about an input (OpenAI Decisions format) - * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * @description Judge an `input` against one or more `questions` and get the answers back in the same order. * * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with * `probabilities`), `score` (an index into `levels`, with `probabilities`) and `box` (the image region the - * `instructions` describe, with `confidence`). Requests that break these rules are rejected with a 400 before - * any provider is called. `model` is any decision model in the proxy - * model list. System One decision models accept this format too, LiteLLM translates the request and the - * answers. Streaming is not supported. + * `instructions` describe, with `confidence`). `model` is any decision model in the proxy model list. OpenAI + * and Azure decision models get the request body as sent, minus LiteLLM settings and with `model` set to the + * deployment's model, and return OpenAI's answers as sent, so any field OpenAI adds works without a LiteLLM + * update. System One decision models accept this format too, LiteLLM translates the request and the answers and + * rejects a request it cannot translate with a 400 before calling the provider. Streaming is not supported. * * [Docs](https://docs.litellm.ai/docs/decisions) */ @@ -40354,10 +40356,13 @@ export interface components { /** Name */ name: string | null; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default box + * @constant */ type: "box"; + } & { + [key: string]: unknown; }; /** OpenAIChoiceAnswer */ OpenAIChoiceAnswer: { @@ -40370,10 +40375,13 @@ export interface components { /** Probabilities */ probabilities: components["schemas"]["OpenAIChoiceProbability"][]; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default choice + * @constant */ type: "choice"; + } & { + [key: string]: unknown; }; /** OpenAIChoiceProbability */ OpenAIChoiceProbability: { @@ -40381,6 +40389,8 @@ export interface components { probability: number; /** Value */ value: string | boolean; + } & { + [key: string]: unknown; }; /** OpenAIDecisionInputTokensDetails */ OpenAIDecisionInputTokensDetails: { @@ -40394,6 +40404,8 @@ export interface components { * @default 0 */ cached_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIDecisionOutputTokensDetails */ OpenAIDecisionOutputTokensDetails: { @@ -40402,11 +40414,13 @@ export interface components { * @default 0 */ reasoning_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIDecisionResponse */ OpenAIDecisionResponse: { /** Answers */ - answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIBoxAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIBoxAnswer"] | components["schemas"]["OpenAIRefusalAnswer"] | components["schemas"]["OpenAIUnknownAnswer"])[]; /** Model */ model: string; usage: components["schemas"]["OpenAIDecisionUsage"]; @@ -40434,6 +40448,8 @@ export interface components { output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; /** Total Tokens */ total_tokens: number; + } & { + [key: string]: unknown; }; /** OpenAIPredicateAnswer */ OpenAIPredicateAnswer: { @@ -40442,20 +40458,26 @@ export interface components { /** Probability */ probability: number; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default predicate + * @constant */ type: "predicate"; + } & { + [key: string]: unknown; }; /** OpenAIRefusalAnswer */ OpenAIRefusalAnswer: { /** Name */ name: string | null; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default refusal + * @constant */ type: "refusal"; + } & { + [key: string]: unknown; }; /** OpenAIScoreAnswer */ OpenAIScoreAnswer: { @@ -40468,10 +40490,13 @@ export interface components { /** Score */ score: number; /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} + * Type + * @default score + * @constant */ type: "score"; + } & { + [key: string]: unknown; }; /** OpenAIScoreProbability */ OpenAIScoreProbability: { @@ -40481,6 +40506,8 @@ export interface components { probability: number; /** Value */ value: number; + } & { + [key: string]: unknown; }; /** OpenAIToolSearchToolParam */ OpenAIToolSearchToolParam: { @@ -40492,6 +40519,15 @@ export interface components { */ type: "tool_search"; }; + /** OpenAIUnknownAnswer */ + OpenAIUnknownAnswer: { + /** Name */ + name?: string | null; + /** Type */ + type: string; + } & { + [key: string]: unknown; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect.