fix(responses): drop undeclared reasoning values and unknown efforts, and round count-route efforts like inference

A row with no declared reasoning modes or contexts is checked against what bundled OpenAI rows declare, a non-reasoning model keeps only summary, an effort outside the known levels drops unless the deployment declares it, Azure o-series runs only the malformed-value checks, and the count route rounds the effort to the deployment's declared levels
This commit is contained in:
mateo-berri 2026-10-29 15:16:41 -07:00
parent ac70835fee
commit e2e7153896
7 changed files with 215 additions and 29 deletions

View file

@ -65,9 +65,7 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
"""
if not drop_params:
return dict(response_api_optional_params)
kept: Final = self.params_dropped_for_deployment(
response_api_optional_params, model, declared_reasoning_efforts=()
)
kept: Final = self._well_formed_params(response_api_optional_params)
if "temperature" in kept:
verbose_logger.debug(
"Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model

View file

@ -35,7 +35,10 @@ from litellm.llms.openai.workload_identity import (
openai_static_api_key,
resolve_openai_workload_identity_config,
)
from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
from litellm.router_utils.reasoning_effort_capability import (
deployment_reasoning_effort_levels,
resolve_deployment_reasoning_effort,
)
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
@ -96,6 +99,24 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
)
def _with_deployment_reasoning(
request: Mapping[str, object], model: str, model_info: object, policy: DropPolicy
) -> Mapping[str, object]:
reasoning: Final = as_str_mapping(request.get("reasoning"))
if reasoning is None:
return request
resolved: Final = resolve_deployment_reasoning_effort(
reasoning,
model=model,
custom_llm_provider=LlmProviders.OPENAI.value,
deployment_model_info=model_info,
model_group=None,
drop_params=policy.drop_params,
).value
without_reasoning: Final = {field: value for field, value in request.items() if field != "reasoning"}
return without_reasoning if resolved is None else {**without_reasoning, "reasoning": resolved}
class OpenAITokenCounter(BaseTokenCounter):
"""Token counter implementation for OpenAI provider using the Responses API."""
@ -219,14 +240,12 @@ class OpenAITokenCounter(BaseTokenCounter):
resolved: Final = _count_credential(deployment, request_model or model_to_use)
if isinstance(resolved, OpenAIError):
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
model_info: Final = None if deployment is None else deployment.get("model_info")
body: Final = native_count_request(
request,
_with_deployment_reasoning(request, model_to_use, model_info, policy),
model_to_use,
policy,
declared_reasoning_efforts=deployment_reasoning_effort_levels(
None if deployment is None else deployment.get("model_info")
)
or (),
declared_reasoning_efforts=deployment_reasoning_effort_levels(model_info) or (),
)
return await self._counted(
resolved,

View file

@ -2,7 +2,7 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from dataclasses import dataclass
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_type_hints
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_args, get_type_hints
import httpx
from openai.types.responses import ResponseReasoningItem
@ -50,6 +50,7 @@ from litellm.responses.utils import openai_well_formed_reasoning
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import *
from litellm.types.llms.openai import REASONING_EFFORT
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
@ -321,6 +322,7 @@ _TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"})
_PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE)
_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT))
def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool:
@ -499,19 +501,27 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return None
def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
if self.custom_llm_provider == LlmProviders.OPENAI and OpenAIGPTConfig.is_openai_catalog_model(model):
return openai_well_formed_reasoning(reasoning, model)
return reasoning
if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model):
return reasoning
return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model))
def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool:
if effort is None or self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES:
return True
return isinstance(effort, str) and (
effort in _REASONING_EFFORTS or effort in (declared_reasoning_efforts or ())
)
def _well_formed_params(self, params: Mapping[str, object]) -> Mapping[str, object]:
if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES:
return params
return {name: value for name, value in params.items() if _is_well_formed(name, value, params)}
def params_dropped_for_deployment(
self, params: Mapping[str, object], model: str, *, declared_reasoning_efforts: Collection[str] | None
) -> Mapping[str, object]:
"""The request params this deployment forwards under drop_params, malformed values removed."""
well_formed: Final = (
{name: value for name, value in params.items() if _is_well_formed(name, value, params)}
if self.custom_llm_provider in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES
else params
)
well_formed: Final = self._well_formed_params(params)
without_reasoning: Final = {name: value for name, value in well_formed.items() if name != "reasoning"}
reasoning: Final = well_formed.get("reasoning")
refusal: Final = self._reasoning_refusal(reasoning, model, declared_reasoning_efforts)
@ -520,7 +530,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
reasoning_fields: Final = as_str_mapping(reasoning)
if reasoning_fields is None:
return well_formed
effort_kept: Final = not isinstance(refusal, _EffortDisabled)
effort_kept: Final = not isinstance(refusal, _EffortDisabled) and self._is_well_formed_effort(
reasoning_fields.get("effort"), declared_reasoning_efforts
)
kept_reasoning: Final = self._accepted_reasoning(
{key: value for key, value in reasoning_fields.items() if effort_kept or key != "effort"}, model
)

View file

@ -2,11 +2,12 @@ import base64
import os
import re
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from functools import reduce
from functools import lru_cache, reduce
from itertools import chain
from types import MappingProxyType
from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import TypedDict as ExtraItemsTypedDict
from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion
@ -14,6 +15,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
@ -1412,6 +1414,7 @@ def reasoning_with_summary_default(
_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
_NON_REASONING_MODEL_FIELD_VALUES: Final = MappingProxyType({"summary": _SUMMARY_VALUES})
def _openai_model_info(model: str) -> ModelInfo | None:
@ -1421,18 +1424,42 @@ def _openai_model_info(model: str) -> ModelInfo | None:
return None if isinstance(model_info, ModelInfoLookupMiss) else model_info
def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | None:
return None if values is None else frozenset(values)
class _ReasoningDeclarations(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
litellm_provider: str | None = None
reasoning_modes: tuple[str, ...] | None = None
reasoning_contexts: tuple[str, ...] | None = None
_BUNDLED_REASONING_DECLARATIONS: Final = TypeAdapter(dict[str, _ReasoningDeclarations])
@lru_cache(maxsize=1)
def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]:
bundled: Final = _BUNDLED_REASONING_DECLARATIONS.validate_json(GetModelCostMap.read_local_model_cost_map_text())
openai_rows: Final = tuple(row for row in bundled.values() if row.litellm_provider == "openai")
return MappingProxyType(
{
"mode": frozenset(chain.from_iterable(row.reasoning_modes or () for row in openai_rows)),
"context": frozenset(chain.from_iterable(row.reasoning_contexts or () for row in openai_rows)),
}
)
def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) -> frozenset[str | None]:
return undeclared if values is None else frozenset(values)
def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Mapping[str, frozenset[str | None] | None]:
if model_info is None:
return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None})
declared_anywhere: Final = _reasoning_values_any_openai_row_declares()
return MappingProxyType(
{
"summary": _SUMMARY_VALUES,
"mode": _declared_values(model_info.get("reasoning_modes")),
"context": _declared_values(model_info.get("reasoning_contexts")),
"mode": _declared_values(model_info.get("reasoning_modes"), declared_anywhere["mode"]),
"context": _declared_values(model_info.get("reasoning_contexts"), declared_anywhere["context"]),
}
)
@ -1467,10 +1494,15 @@ def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> M
return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model))
def openai_well_formed_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
return _reasoning_with_accepted_fields(
reasoning, _openai_declared_reasoning_field_values(_openai_model_info(model))
def openai_well_formed_reasoning(
reasoning: Mapping[str, object], model: str, *, model_reasons: bool
) -> Mapping[str, object]:
values_by_field: Final = (
_openai_declared_reasoning_field_values(_openai_model_info(model))
if model_reasons
else _NON_REASONING_MODEL_FIELD_VALUES
)
return _reasoning_with_accepted_fields(reasoning, values_by_field)
def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:

View file

@ -667,3 +667,24 @@ async def test_the_count_reads_the_deployments_declared_efforts_under_drop_param
sent: Final = json.loads(route.calls.last.request.content)
assert ("reasoning" in sent) is effort_counted
@pytest.mark.asyncio
@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "send-as-is"])
async def test_the_count_rounds_the_effort_to_a_declared_level_as_inference_does(
local_model_cost_map, monkeypatch: pytest.MonkeyPatch, drop_params: bool
) -> None:
declared: Final = ["low", "medium", "high"]
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9}))
await OpenAITokenCounter().count_responses_request(
model_to_use="gpt-6.1-sol",
request={"model": "my-gpt", "input": "hi", "reasoning": {"effort": "max", "summary": "auto"}},
policy=DropPolicy(drop_params=drop_params, additional_drop_params=frozenset()),
deployment={"litellm_params": {"api_key": "sk-test"}, "model_info": {"reasoning_effort_levels": declared}},
request_model="my-gpt",
)
sent: Final = json.loads(route.calls.last.request.content)
assert sent["reasoning"] == {"effort": "high", "summary": "auto"}

View file

@ -18,6 +18,7 @@ from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIC
from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
@ -3336,6 +3337,18 @@ class TestMalformedParamsFollowDropParams:
assert malformed not in dropped
assert sent == params
def test_azure_o_series_forwards_a_row_disabled_effort_while_dropping_malformed_values(
self, local_model_cost_map, monkeypatch
):
row: Final = litellm.model_cost["gpt-6.1-sol"]
monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**row, "supports_minimal_reasoning_effort": False})
mapped: Final = AzureOpenAIOSeriesResponsesAPIConfig().map_openai_params(
{"reasoning": {"effort": "minimal"}, "tool_choice": "required"}, "gpt-6.1-sol", True
)
assert mapped == {"reasoning": {"effort": "minimal"}}
@pytest.mark.parametrize(
"params",
[
@ -3383,6 +3396,68 @@ class TestMalformedParamsFollowDropParams:
assert dropped["reasoning"] == {key: kept for key, kept in declared.items() if key != field}
assert sent["reasoning"] == reasoning
@pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")])
def test_a_row_without_reasoning_declarations_drops_values_no_openai_row_declares(
self, local_model_cost_map, monkeypatch, field, declarations
):
monkeypatch.setattr(litellm, "drop_params", False)
undeclared_model: Final = next(
name
for name, row in litellm.model_cost.items()
if row.get("litellm_provider") == "openai" and row.get("supports_reasoning") and declarations not in row
)
declared_elsewhere: Final = litellm.model_cost["gpt-6.1-sol"][declarations][0]
malformed: Final = {"summary": "auto", field: "not-a-declared-value"}
declared: Final = {"summary": "auto", field: declared_elsewhere}
config: Final = OpenAIResponsesAPIConfig()
dropped: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, True)
sent: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, False)
kept: Final = config.map_openai_params({"reasoning": dict(declared)}, undeclared_model, True)
assert dropped["reasoning"] == {"summary": "auto"}
assert sent["reasoning"] == malformed
assert kept["reasoning"] == declared
def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params(self, local_model_cost_map, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
non_reasoning_model: Final = next(
name
for name, row in litellm.model_cost.items()
if row.get("litellm_provider") == "openai"
and not row.get("supports_reasoning")
and OpenAIGPTConfig.is_openai_catalog_model(name)
)
request: Final = {"reasoning": {"summary": "auto", "mode": "standard", "context": "auto"}, "tool_choice": "auto"}
config: Final = OpenAIResponsesAPIConfig()
dropped: Final = config.map_openai_params(dict(request), non_reasoning_model, True)
sent: Final = config.map_openai_params(dict(request), non_reasoning_model, False)
assert dropped == {"reasoning": {"summary": "auto"}, "tool_choice": "auto"}
assert sent == request
@pytest.mark.parametrize(
("config", "model"),
[(OpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol")],
ids=["openai", "azure"],
)
def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it(
self, local_model_cost_map, monkeypatch, config, model
):
monkeypatch.setattr(litellm, "drop_params", False)
reasoning: Final = {"effort": "not-an-effort", "summary": "auto"}
dropped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True)
sent: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, False)
declared: Final = config.map_openai_params_for_deployment(
{"reasoning": dict(reasoning)}, model, True, declared_reasoning_efforts=("not-an-effort",)
)
assert dropped["reasoning"] == {"summary": "auto"}
assert sent["reasoning"] == reasoning
assert declared["reasoning"] == reasoning
@pytest.mark.parametrize(
("config", "model"),
[(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (OpenAIResponsesAPIConfig(), "my-fine-tuned-model")],

View file

@ -10,7 +10,8 @@ import litellm
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils, openai_bridge_reasoning
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -1009,3 +1010,31 @@ def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate():
else:
litellm.model_cost[model] = previous_row
_invalidate_model_cost_lowercase_map()
@pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")])
def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_drop_params(
local_model_cost_map, field, declarations
):
undeclared_model: Final = next(
name
for name, row in litellm.model_cost.items()
if row.get("litellm_provider") == "openai"
and row.get("supports_reasoning")
and declarations not in row
and OpenAIGPTConfig.is_openai_catalog_model(name)
)
requested: Final = {"effort": "high", "summary": "auto", field: "not-a-declared-value"}
def bridged(drop_params: bool) -> object:
return openai_bridge_reasoning(
requested=requested,
effort_kept=True,
summary_alias=None,
model=undeclared_model,
drop_params=drop_params,
additional_drop_params=None,
)
assert bridged(True) == {"effort": "high", "summary": "auto"}
assert bridged(False) == requested