From c0ded7b2824e7af9d8e5e8008fdf35d1afec0c3c Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:47:59 -0700 Subject: [PATCH] fix(responses)!: send a dict reasoning_effort the same way on every Responses bridge (#49082) * fix(responses): send a dict reasoning_effort the same way on every Responses bridge Azure and ChatGPT bridges dropped reasoning mode and context unless the dict also carried summary. Every bridge now runs the reasoning step the OpenAI bridge uses, and each Responses config declares the reasoning values its API accepts, so drop_params drops what a backend refuses. Perplexity ignores mode and context, so it answers 400 for them without drop_params. * fix(responses): refuse ignored reasoning fields on xAI and Fireworks, resolve openai/ names for OpenRouter * fix(responses): refuse a reasoning mode Copilot ignores, keep catalog modes, return refusals as values Apply additional_drop_params before the refusal check, keep a null reasoning field instead of refusing it, let a cost-map row's reasoning_modes and reasoning_contexts override a backend's declared values, and declare GitHub Copilot's accepted values from a live probe * fix(responses): read Azure base_model and deployment rows, accept standard mode, refuse only mode and context * fix(responses): send only effort and summary to an Azure deployment litellm cannot identify under drop_params Azure and Azure AI deployment names are user-chosen, so when neither the name, model_info.base_model, nor a cost map row resolves the model, drop_params now drops mode and context instead of forwarding a pro the model may refuse. Names resolve by their last path segment, so a regional base_model like azure/eu/gpt-5.4 follows gpt-5.4's rows * fix(responses): resolve deployment names by model family and keep reasoning_modes per deployment A custom Azure deployment name now resolves to the longest OpenAI catalog prefix of its base_model or name, an unresolved name forwards mode and context, and a deployment's model_info reasoning_modes and reasoning_contexts apply only to that deployment instead of every sibling sharing its model name. litellm_proxy forwards every reasoning field. * fix(responses): find the model family anywhere in an Azure deployment name and keep only effort and summary for unknown gpt-5 or gpt-6 names under drop_params * fix(responses): look up OpenAI model names inside deployment names only on Azure Databricks endpoints like databricks-gpt-5-5-pro spell versions with hyphens and other backends name local models freely, so only Azure and Azure AI search inside the name, and a run that cuts a hyphen-spelled version never matches * fix(responses): judge Azure AI reasoning fields by the request api_base --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/azure/responses/transformation.py | 7 + .../llms/base_llm/responses/transformation.py | 28 +- .../llms/chatgpt/responses/transformation.py | 4 + .../fireworks_ai/responses/transformation.py | 10 + .../responses/transformation.py | 7 + .../litellm_proxy/responses/transformation.py | 4 + .../perplexity/responses/transformation.py | 11 + litellm/llms/xai/responses/transformation.py | 10 + litellm/main.py | 26 +- litellm/responses/utils.py | 186 +++++-- litellm/types/utils.py | 2 + tests/unit/test_main.py | 457 ++++++++++++++++++ 12 files changed, 716 insertions(+), 36 deletions(-) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 09777da8669..2b757bb10d9 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * from litellm.types.responses.main import * @@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def _effort_resolves_to_none(model: str, effort: str | None) -> bool: return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort) + def deployment_names_wrap_openai_models(self) -> bool: + return True + + def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None: + return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None + def get_supported_openai_params(self, model: str) -> list: """ Azure Responses API does not support context_management (compaction). diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 792b14090b8..32d0e408aba 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args import httpx @@ -27,6 +27,20 @@ else: BaseLLMException = Any ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"] +ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None] +REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) +SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {"summary": REASONING_SUMMARY_VALUES} +) +STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})} +) +STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None} +) +ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None} +) class BaseResponsesAPIConfig(ABC): @@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC): """The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for.""" return "detailed" + def reasoning_field_values(self) -> ReasoningFieldValues | None: + return None + + def deployment_names_wrap_openai_models(self) -> bool: + return False + + def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None: + return None + + def ignores_unsupported_reasoning_fields(self) -> bool: + return False + @abstractmethod def map_openai_params( self, diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 72b5cde21d1..26a5a3f150b 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo safe_convert_created_field, ) from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """ChatGPT does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_REASONING_FIELD_VALUES diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index 86d9df9618a..463b9cbfc9a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -6,6 +6,10 @@ from urllib.parse import unquote import httpx from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import ( + STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, + ReasoningFieldValues, +) from litellm.llms.fireworks_ai.common_utils import ( FIREWORKS_FORWARD_USER_ID_PARAM, get_fireworks_forwarded_user_id, @@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_file_search(self) -> bool: return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index b055d388dd5..b87a3d543e7 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -24,6 +24,7 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ( ResponseInputParam, @@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): return False + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True + def supports_native_websocket(self) -> bool: """GitHub Copilot does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index fc94eb30504..ed62e132c99 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders @@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.LITELLM_PROXY + def reasoning_field_values(self) -> ReasoningFieldValues: + return ANY_REASONING_FIELD_VALUES + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 58f71028451..afd27b0f696 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -9,6 +9,7 @@ The only provider quirks: Ref: https://docs.perplexity.ai/api-reference/responses-post """ +from types import MappingProxyType from typing import Any, Final import httpx @@ -17,6 +18,7 @@ from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse @@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders _DECODED_JSON: Final = TypeAdapter(object) +PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType( + {**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})} +) class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): @@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """Perplexity does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return PERPLEXITY_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index b6ea516fb79..c907fde9926 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -10,6 +10,10 @@ from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import ( + STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, + ReasoningFieldValues, +) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.secret_managers.main import get_secret_str @@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """XAI does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/main.py b/litellm/main.py index 8d6e07d6a8f..54afe654cec 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6267,6 +6267,9 @@ def completion( # reasoningSummary/reasoning_summary without tools (AI SDK) that the first # (early) check doesn't cover. _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params) + typed_deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) if responses_api_model_info.get("mode") != "responses": responses_api_model_info, model = responses_api_bridge_check( model=model, @@ -6276,9 +6279,7 @@ def completion( reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")), reasoning_summary=_reasoning_summary_for_bridge, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=typed_deployment_model_info, ) # Use base_model (the true underlying model) for Azure model-type @@ -6290,19 +6291,28 @@ def completion( optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) - if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict): - from litellm.responses.utils import openai_bridge_reasoning + if isinstance(resolved_reasoning_effort, dict): + from litellm.responses.utils import ( + RefusedReasoningFields, + bridge_reasoning, + raise_refused_reasoning_fields, + ) - optional_params["reasoning_effort"] = openai_bridge_reasoning( + bridged_reasoning: Final = bridge_reasoning( requested=resolved_reasoning_effort, effort_kept="reasoning_effort" in optional_params, summary_alias=rs_val, model=model, + base_model=base_model if isinstance(base_model, str) else None, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + deployment_model_info=typed_deployment_model_info, drop_params=drop_unsupported_reasoning, additional_drop_params=routing_kwargs.get("additional_drop_params"), ) - elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: - optional_params["reasoning_effort"] = resolved_reasoning_effort + if isinstance(bridged_reasoning, RefusedReasoningFields): + raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider) + optional_params["reasoning_effort"] = bridged_reasoning elif rs_val is not None: eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) if isinstance(eff, dict): diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index b0db0516658..1a97c1cb48c 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,10 +1,11 @@ import base64 import os import re -from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import reduce from types import MappingProxyType -from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload +from typing import Any, Final, NoReturn, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict as ExtraItemsTypedDict @@ -15,7 +16,13 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel +from litellm.llms.base_llm.responses.transformation import ( + ANY_REASONING_FIELD_VALUES, + REASONING_SUMMARY_VALUES, + BaseResponsesAPIConfig, + ReasoningFieldValues, + ReasoningSummaryLevel, +) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -1411,9 +1418,6 @@ def reasoning_with_summary_default( return {**reasoning} if summary is None else {**reasoning, "summary": summary} -_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) - - def _openai_model_info(model: str) -> ModelInfo | None: from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration @@ -1425,24 +1429,115 @@ def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | No return None if values is None else frozenset(values) -def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]: +_REFUSABLE_REASONING_FIELDS: Final = frozenset({"mode", "context"}) + + +def _openai_reasoning_field_values(model: str) -> ReasoningFieldValues: model_info: Final = _openai_model_info(model) if model_info is None: - return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) + return ANY_REASONING_FIELD_VALUES if not model_info.get("supports_reasoning"): return MappingProxyType({}) return MappingProxyType( { - "summary": _SUMMARY_VALUES, + "summary": REASONING_SUMMARY_VALUES, "mode": _declared_values(model_info.get("reasoning_modes")), "context": _declared_values(model_info.get("reasoning_contexts")), } ) -def _openai_accepts_reasoning_field( - key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None] -) -> bool: +_CATALOG_REASONING_FIELDS: Final = (("mode", "reasoning_modes"), ("context", "reasoning_contexts")) + + +def _catalog_values(values: object) -> frozenset[str | None]: + return frozenset(value for value in iter_items(values) if isinstance(value, str)) + + +def _declared_reasoning_values(row: Mapping[str, object] | None) -> Mapping[str, frozenset[str | None]]: + if row is None: + return {} + return { + field: _catalog_values(row[key]) for field, key in _CATALOG_REASONING_FIELDS if isinstance(row.get(key), list) + } + + +def _with_declared_reasoning_values( + defaults: ReasoningFieldValues | None, + model: str, + custom_llm_provider: str, + deployment_model_info: Mapping[str, object] | None, +) -> ReasoningFieldValues | None: + catalog: Final = cast(Mapping[str, object], litellm.model_cost) # cast-ok: read-only view of the legacy registry + catalog_row: Final = as_str_mapping(catalog.get(f"{custom_llm_provider}/{model.removeprefix('responses/')}")) + overrides: Final = { + **_declared_reasoning_values(catalog_row), + **_declared_reasoning_values(deployment_model_info), + } + if not overrides: + return defaults + return MappingProxyType({**(ANY_REASONING_FIELD_VALUES if defaults is None else defaults), **overrides}) + + +def _is_version_number(part: str) -> bool: + return part.isdigit() and len(part) <= 2 + + +def _splits_a_version(parts: Sequence[str], end: int) -> bool: + return end < len(parts) and _is_version_number(parts[end - 1]) and _is_version_number(parts[end]) + + +def _hyphen_runs_longest_first(name: str) -> Iterator[str]: + parts: Final = name.rsplit("/", 1)[-1].split("-") + for length in range(len(parts), 0, -1): + yield from ( + "-".join(parts[start : start + length]) + for start in range(len(parts) - length + 1) + if not _splits_a_version(parts, start + length) + ) + + +def _openai_catalog_family(name: str) -> str | None: + runs: Final = _hyphen_runs_longest_first(name) + return next((run for run in runs if OpenAIGPTConfig.is_openai_catalog_model(run)), None) + + +def _openai_catalog_name(name: str) -> str | None: + last_segment: Final = name.rsplit("/", 1)[-1] + return last_segment if OpenAIGPTConfig.is_openai_catalog_model(last_segment) else None + + +def _openai_catalog_model(model: str, base_model: str | None, names_wrap_models: bool) -> str | None: + find: Final = _openai_catalog_family if names_wrap_models else _openai_catalog_name + families: Final = (find(name) for name in (base_model, model) if name) + return next((family for family in families if family is not None), None) + + +def _default_reasoning_field_values( + model: str, base_model: str | None, responses_config: BaseResponsesAPIConfig | None +) -> ReasoningFieldValues | None: + declared: Final = None if responses_config is None else responses_config.reasoning_field_values() + if declared is not None: + return declared + names_wrap_models: Final = responses_config is not None and responses_config.deployment_names_wrap_openai_models() + catalog_model: Final = _openai_catalog_model(model, base_model, names_wrap_models) + if catalog_model is not None: + return _openai_reasoning_field_values(catalog_model) + return None if responses_config is None else responses_config.unresolved_model_reasoning_field_values(model) + + +def _reasoning_field_values( + model: str, + base_model: str | None, + custom_llm_provider: str, + responses_config: BaseResponsesAPIConfig | None, + deployment_model_info: Mapping[str, object] | None, +) -> ReasoningFieldValues | None: + defaults: Final = _default_reasoning_field_values(model, base_model, responses_config) + return _with_declared_reasoning_values(defaults, model, custom_llm_provider, deployment_model_info) + + +def _accepts_reasoning_field(key: str, value: object, values_by_field: ReasoningFieldValues) -> bool: if key == "effort": return True if key not in values_by_field: @@ -1451,11 +1546,8 @@ def _openai_accepts_reasoning_field( return allowed is None or (isinstance(value, str | None) and value in allowed) -def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - values_by_field: Final = _openai_reasoning_field_values(model) - return { - key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field) - } +def _accepted_reasoning(reasoning: Mapping[str, object], values_by_field: ReasoningFieldValues) -> Mapping[str, object]: + return {key: value for key, value in reasoning.items() if _accepts_reasoning_field(key, value, values_by_field)} def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: @@ -1464,26 +1556,66 @@ def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str)) -def openai_bridge_reasoning( +@dataclass(frozen=True, slots=True) +class RefusedReasoningFields: + refused: tuple[str, ...] + + +def raise_refused_reasoning_fields(refusal: RefusedReasoningFields, model: str, custom_llm_provider: str) -> NoReturn: + raise litellm.UnsupportedParamsError( + message=( + f"{custom_llm_provider} does not support parameters: " + f"{[f'reasoning_effort.{key}' for key in refusal.refused]}, for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=custom_llm_provider, + model=model, + ) + + +def bridge_reasoning( *, requested: Mapping[str, object], effort_kept: bool, summary_alias: object, model: str, + base_model: str | None, + custom_llm_provider: str, + api_base: str | None, + deployment_model_info: Mapping[str, object] | None, drop_params: bool, additional_drop_params: object, -) -> Mapping[str, object] | None: +) -> Mapping[str, object] | RefusedReasoningFields | None: + from litellm.utils import ProviderConfigManager + drop_paths: Final = _additional_drop_paths(additional_drop_params) if "reasoning_effort" in drop_paths: return None kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"} - with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai") - accepted: Final = ( - _openai_accepted_reasoning(with_summary, model) - if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model) - else {**with_summary} + with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, custom_llm_provider) + sent: Final = ( + as_str_mapping( + _apply_nested_drop_params({"reasoning_effort": {**with_summary}}, list(drop_paths)).get("reasoning_effort") + ) + or {} ) - remaining: Final = as_str_mapping( - _apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort") + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + custom_llm_provider, model, api_base ) - return remaining or None + values_by_field: Final = _reasoning_field_values( + model, base_model, custom_llm_provider, responses_config, deployment_model_info + ) + accepted: Final = sent if values_by_field is None else _accepted_reasoning(sent, values_by_field) + refused: Final = tuple( + key + for key, value in sent.items() + if key in _REFUSABLE_REASONING_FIELDS and key not in accepted and value is not None + ) + if ( + refused + and not drop_params + and responses_config is not None + and responses_config.ignores_unsupported_reasoning_fields() + ): + return RefusedReasoningFields(refused) + return (accepted if drop_params else sent) or None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5905aaa3380..6a758961c7b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4155,6 +4155,8 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_assistant_prefill", "supports_bedrock_native_structured_output", "supports_pdf_input", + "reasoning_modes", + "reasoning_contexts", } ) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index b2ae6b43d26..b0bf1cb879e 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -2463,6 +2463,463 @@ _BRIDGED_RESPONSE_ANSWER: Final = json.dumps( ).encode() +@pytest.fixture +def non_openai_responses_bridge( + restore_model_registry, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> Callable[[str], tuple[str, Mapping[str, str], respx.Route]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False) + for chatgpt_env in ("CHATGPT_AUTH_FILE", "CHATGPT_API_BASE", "OPENAI_CHATGPT_API_BASE"): + monkeypatch.delenv(chatgpt_env, raising=False) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_text( + json.dumps({"access_token": "token", "account_id": "acct", "expires_at": 4102444800}) + ) + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "api-key.json").write_text( + json.dumps({"token": "token", "expires_at": 4102444800, "endpoints": {"api": "https://api.githubcopilot.com"}}) + ) + litellm.register_model( + { + **{ + model_name: { + "litellm_provider": "openai", + "mode": "chat", + "supports_reasoning": True, + "reasoning_modes": ["standard"], + "reasoning_contexts": ["auto", "current_turn", "all_turns"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for model_name in ("gpt-6", "gpt-6.1-standard-only") + }, + **{ + f"{provider}/{model_name}": { + "litellm_provider": provider, + "mode": "responses", + "supports_reasoning": True, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for provider, model_name in ( + ("chatgpt", "gpt-6.1-bridge-test"), + ("perplexity", "bridge-test-model"), + ("github_copilot", "gpt-6.1-bridge-test"), + ) + }, + }, + persist_across_reloads=False, + ) + backends: Final = { + "azure": ( + "azure/responses/gpt-6.1-standard-only", + "https://test.openai.azure.com/", + {"api_base": "https://test.openai.azure.com", "api_key": "fake-key"}, + ), + "azure_ai": ( + "azure_ai/responses/gpt-6.1-standard-only-prod", + _FOUNDRY_API_BASE, + {"api_base": _FOUNDRY_API_BASE, "api_key": "fake-key"}, + ), + "chatgpt": ("chatgpt/gpt-6.1-bridge-test", "https://chatgpt.com/backend-api/codex/responses", {}), + "github_copilot": ("github_copilot/gpt-6.1-bridge-test", "https://api.githubcopilot.com/responses", {}), + "databricks": ( + "databricks/responses/databricks-gpt-bridge-test", + "https://dbx.example/serving-endpoints/responses", + {"api_base": "https://dbx.example/serving-endpoints", "api_key": "fake-key"}, + ), + "perplexity": ( + "perplexity/bridge-test-model", + "https://api.perplexity.ai/v1/responses", + {"api_key": "fake-key"}, + ), + "xai": ("xai/responses/grok-bridge-test", "https://api.x.ai/v1/responses", {"api_key": "fake-key"}), + "fireworks_ai": ( + "fireworks_ai/responses/accounts/fireworks/models/bridge-test", + "https://api.fireworks.ai/inference/v1/responses", + {"api_key": "fake-key"}, + ), + "openrouter": ( + "openrouter/responses/openai/gpt-6.1-standard-only", + "https://openrouter.ai/api/v1/responses", + {"api_key": "fake-key"}, + ), + "litellm_proxy": ( + "litellm_proxy/responses/gpt-6.1-standard-only", + "https://proxy.example/v1/responses", + {"api_base": "https://proxy.example/v1", "api_key": "fake-key"}, + ), + } + + def route_backend(backend: str) -> tuple[str, Mapping[str, str], respx.Route]: + model, url, kwargs = backends[backend] + route: Final = respx_mock.post(url__startswith=url).respond( + content=_BRIDGED_RESPONSE_ANSWER, headers={"content-type": "application/json"} + ) + return model, kwargs, route + + return route_backend + + +@pytest.mark.parametrize( + ("backend", "reasoning_effort"), + [ + ("azure", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("chatgpt", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("github_copilot", {"effort": "low", "mode": "standard", "context": "all_turns"}), + ("databricks", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("openrouter", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("perplexity", {"effort": "low", "mode": "standard", "context": "all_turns", "generate_summary": "auto"}), + ("xai", {"effort": "low", "mode": "standard", "generate_summary": "auto"}), + ("fireworks_ai", {"effort": "low", "mode": "standard", "generate_summary": "auto"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_non_openai_responses_bridge_sends_the_whole_reasoning_effort_dict_with_or_without_summary( + backend: str, + reasoning_effort: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + requested: Final = {**reasoning_effort, **summary} + + litellm.completion( + model=model, messages=[{"role": "user", "content": "hello"}], reasoning_effort=requested, **kwargs + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == requested + + +@pytest.mark.parametrize( + ("backend", "reasoning_effort", "expected_reasoning"), + [ + ("azure", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low", "context": "all_turns"}), + ( + "chatgpt", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "chatgpt", + {"effort": "low", "mode": "standard", "context": "current_turn"}, + {"effort": "low", "mode": "standard", "context": "current_turn"}, + ), + ( + "github_copilot", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "openrouter", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "perplexity", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ("perplexity", {"effort": "low", "mode": "standard", "context": "every_turn"}, {"effort": "low", "mode": "standard"}), + ("xai", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low"}), + ("fireworks_ai", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_non_openai_responses_bridge_drops_reasoning_fields_the_backend_refuses_under_drop_params( + backend: str, + reasoning_effort: dict[str, str], + expected_reasoning: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={**reasoning_effort, **summary}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {**expected_reasoning, **summary} + + +@pytest.mark.parametrize( + ("backend", "ignored"), + [ + ("perplexity", {"mode": "pro"}), + ("perplexity", {"context": "every_turn"}), + ("xai", {"context": "all_turns"}), + ("fireworks_ai", {"context": "all_turns"}), + ("github_copilot", {"mode": "pro"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_responses_bridge_rejects_reasoning_fields_the_backend_ignores_without_drop_params( + backend: str, + ignored: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"reasoning_effort\.{next(iter(ignored))}") as raised: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", **ignored, **summary}, + **kwargs, + ) + + assert raised.value.status_code == 400 + assert route.called is False + + +@pytest.mark.parametrize("backend", ["perplexity", "xai", "fireworks_ai"]) +def test_responses_bridge_sends_the_rest_when_additional_drop_params_removes_the_refused_fields( + backend: str, + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns", "summary": "auto"}, + additional_drop_params=["reasoning_effort.mode", "reasoning_effort.context"], + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "summary": "auto"} + + +@pytest.mark.parametrize("backend", ["perplexity", "xai", "fireworks_ai"]) +def test_responses_bridge_forwards_a_null_reasoning_field_instead_of_rejecting_it( + backend: str, + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": None}, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "mode": None} + + +def test_chatgpt_bridge_keeps_a_reasoning_mode_its_catalog_row_declares_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("chatgpt") + litellm.register_model( + { + "chatgpt/gpt-6.1-pro-bridge-test": { + "litellm_provider": "chatgpt", + "mode": "responses", + "supports_reasoning": True, + "reasoning_modes": ["pro"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + litellm.completion( + model="chatgpt/gpt-6.1-pro-bridge-test", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "mode": "pro"} + + +@pytest.mark.parametrize( + ("model", "model_info"), + [ + ("azure/responses/custom-deployment-name", {"base_model": "azure/gpt-6.1-standard-only"}), + ("azure/responses/custom-deployment-name", {"base_model": "azure/eu/gpt-6.1-standard-only"}), + ("azure/responses/custom-deployment-name", {"base_model": "azure/gpt-6.1-standard-only-2026-07-09"}), + ("azure/responses/gpt-6.1-standard-only-prod", {}), + ("azure/responses/eu-gpt-6.1-standard-only-prod", {}), + ("azure/responses/gpt-6-2026-07-09", {}), + ], +) +def test_azure_bridge_reads_reasoning_modes_from_the_deployment_model_family_under_drop_params( + model: str, + model_info: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + model_info=model_info, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + +def test_azure_ai_bridge_judges_reasoning_fields_by_the_request_api_base_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("AZURE_AI_API_BASE", "https://legacy-models.eastus2.inference.ai.azure.com") + model, kwargs, route = non_openai_responses_bridge("azure_ai") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + +@pytest.mark.parametrize("model", ["azure/responses/custom-gpt-6x", "azure/responses/gpt-6-1-sol"]) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_azure_bridge_sends_only_effort_and_summary_to_an_unknown_gpt_series_deployment_under_drop_params( + model: str, + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns", **summary}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", **summary} + + +@pytest.mark.parametrize( + ("backend", "model"), + [ + ("azure", "azure/responses/terra-prod"), + ("databricks", "databricks/responses/databricks-gpt-bridge-test"), + ("databricks", "databricks/responses/databricks-gpt-6.1-standard-only"), + ("litellm_proxy", "litellm_proxy/responses/gpt-6.1-standard-only"), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_responses_bridge_forwards_mode_and_context_it_cannot_judge_under_drop_params( + backend: str, + model: str, + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge(backend) + requested: Final = {"effort": "low", "mode": "pro", "context": "all_turns", **summary} + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == requested + + +def test_azure_bridge_applies_a_deployment_reasoning_modes_only_to_that_deployment_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + litellm.register_model( + { + "gpt-6.1-pro-capable": { + "litellm_provider": "openai", + "mode": "chat", + "supports_reasoning": True, + "reasoning_modes": ["standard", "pro"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + litellm_params: Final = {"model": "azure/responses/gpt-6.1-pro-capable", **kwargs} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "standard-only", + "litellm_params": litellm_params, + "model_info": {"reasoning_modes": ["standard"]}, + }, + {"model_name": "pro-capable", "litellm_params": litellm_params}, + ] + ) + requested: Final = {"effort": "low", "mode": "pro"} + + for model_name in ("standard-only", "pro-capable"): + router.completion( + model=model_name, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + ) + litellm.completion( + model="azure/responses/gpt-6.1-pro-capable", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + **kwargs, + ) + + assert [json.loads(call.request.content)["reasoning"] for call in route.calls] == [ + {"effort": "low"}, + requested, + requested, + ] + + +def test_azure_bridge_reads_reasoning_modes_from_the_deployment_cost_map_row_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + litellm.register_model( + { + "azure/custom-standard-deployment": { + "litellm_provider": "azure", + "mode": "responses", + "supports_reasoning": True, + "reasoning_modes": ["standard"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + litellm.completion( + model="azure/responses/custom-standard-deployment", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + def test_responses_bridge_fake_stream_is_bounded_by_the_request_timeout_not_the_stream_timeout( restore_model_registry: None, ) -> None: