From fb2a70e4639b3a79ab5d35241db107c85ded0bbe Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:22:59 -0700 Subject: [PATCH 1/9] fix(responses): drop malformed tool_choice, parallel_tool_calls, previous_response_id and reasoning values under drop_params --- .../responses/o_series_transformation.py | 16 +- .../responses/count_tokens/token_counter.py | 11 +- .../responses/count_tokens/transformation.py | 20 ++- .../llms/openai/responses/transformation.py | 155 ++++++++++++++---- litellm/responses/utils.py | 27 ++- .../count_tokens/test_token_counter.py | 25 +++ ...test_openai_count_tokens_transformation.py | 54 +++++- .../test_openai_responses_transformation.py | 95 +++++++++++ 8 files changed, 346 insertions(+), 57 deletions(-) diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 06346a300e4..259a94bd14d 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -60,19 +60,19 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): """ Map OpenAI parameters for Azure OpenAI O-series Responses API. - Drops temperature parameter if drop_params is True since O-series models + Drops temperature and malformed values if drop_params is True since O-series models don't support temperature in the responses API. """ - mapped_params: Final = dict(response_api_optional_params) - - # If drop_params is enabled, remove temperature parameter for O-series models - if drop_params and "temperature" in mapped_params: + if not drop_params: + return dict(response_api_optional_params) + kept: Final = self.params_dropped_for_deployment( + response_api_optional_params, model, declared_reasoning_efforts=() + ) + if "temperature" in kept: verbose_logger.debug( "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model ) - mapped_params.pop("temperature", None) - - return mapped_params + return {name: value for name, value in kept.items() if name != "temperature"} def is_o_series_model(self, model: str) -> bool: """ diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..88ce5002803 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -35,6 +35,7 @@ from litellm.llms.openai.workload_identity import ( openai_static_api_key, resolve_openai_workload_identity_config, ) +from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests @@ -218,7 +219,15 @@ class OpenAITokenCounter(BaseTokenCounter): resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) - body: Final = native_count_request(request, model_to_use, policy) + body: Final = native_count_request( + request, + model_to_use, + policy, + declared_reasoning_efforts=deployment_reasoning_effort_levels( + None if deployment is None else deployment.get("model_info") + ) + or (), + ) return await self._counted( resolved, lambda api_key, api_base: openai_count_tokens_handler.post_count_request(body, api_key, api_base), diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index a45a0448326..9eb8279ff0b 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -5,7 +5,7 @@ This module handles the transformation of requests to OpenAI's /v1/responses/inp """ import copy -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from typing import Final, Literal, Protocol, TypeAlias from openai.types.responses.input_token_count_params import InputTokenCountParams @@ -20,6 +20,7 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input @@ -400,8 +401,21 @@ def _restored(field: str, value: object, policy: DropPolicy) -> object: return value -def native_count_request(request: Mapping[str, object], model: str, policy: DropPolicy) -> Mapping[str, object]: +def native_count_request( + request: Mapping[str, object], + model: str, + policy: DropPolicy, + *, + declared_reasoning_efforts: Collection[str], +) -> Mapping[str, object]: forwarded: Final = { field: _restored(field, value, policy) for field, value in request.items() if _forwards(field, policy) } - return {**forwarded, "model": model} + kept: Final = ( + OpenAIResponsesAPIConfig().params_dropped_for_deployment( + forwarded, model, declared_reasoning_efforts=declared_reasoning_efforts + ) + if policy.drop_params + else forwarded + ) + return {**kept, "model": model} diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 45fc1fc3ff7..2c69d160308 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,7 +1,8 @@ from collections.abc import Callable, Collection, Mapping, Sequence +from dataclasses import dataclass from functools import lru_cache from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_type_hints import httpx from openai.types.responses import ResponseReasoningItem @@ -11,7 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api -from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, @@ -29,6 +30,7 @@ from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, unsupported_reasoning_effort_message, ) +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE from litellm.responses.litellm_completion_transformation.reasoning_items import is_litellm_minted_reasoning_item from litellm.responses.litellm_completion_transformation.tool_search import ( @@ -38,6 +40,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( tools_with_anthropic_tool_search_mapped, tools_without_anthropic_tool_search, ) +from litellm.responses.utils import openai_well_formed_reasoning from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * @@ -240,6 +243,46 @@ def _bundled_openai_reasoning_models() -> frozenset[str]: ) +_TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"}) + + +_PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE) + + +def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: + if tool_choice is None or params.get("prompt") is not None: + return True + tools: Final = params.get("tools") + if isinstance(tool_choice, str): + return tool_choice in _TOOL_CHOICES_WITHOUT_TOOLS or (tool_choice == "required" and bool(tools)) + return bool(tools) + + +def _is_well_formed(name: str, value: object, params: Mapping[str, object]) -> bool: + match name: + case "tool_choice": + return _accepts_tool_choice(value, params) + case "parallel_tool_calls": + return value is None or isinstance(value, bool) + case "previous_response_id": + return value is None or (isinstance(value, str) and value != "") + case _: + return True + + +@dataclass(frozen=True, slots=True) +class _ReasoningUnsupported: + pass + + +@dataclass(frozen=True, slots=True) +class _EffortDisabled: + effort: str + + +_ReasoningRefusal: TypeAlias = _ReasoningUnsupported | _EffortDisabled + + class _DeleteResponseBody(TypedDict): """Decoded body of the Responses API delete call.""" @@ -354,8 +397,60 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) return effort is not None + def _reasoning_refusal( + self, reasoning: object, model: str, declared_reasoning_efforts: Collection[str] + ) -> _ReasoningRefusal | None: + if ( + self.custom_llm_provider == LlmProviders.OPENAI + and self._requests_reasoning_effort(reasoning) + and not self._supports_reasoning_param(model=model) + ): + return _ReasoningUnsupported() + lookup_name: Final = self._model_map_lookup_name(model) + effort: Final = (as_str_mapping(reasoning) or {}).get("effort") + if ( + isinstance(effort, str) + and effort not in declared_reasoning_efforts + and self._is_gpt_5_model(model=lookup_name) + and self._effort_level_is_disabled(lookup_name, effort) + ): + return _EffortDisabled(effort) + return None + + def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: + if self.custom_llm_provider == LlmProviders.OPENAI and OpenAIGPTConfig.is_openai_catalog_model(model): + return openai_well_formed_reasoning(reasoning, model) + return reasoning + + def params_dropped_for_deployment( + self, params: Mapping[str, object], model: str, *, declared_reasoning_efforts: Collection[str] + ) -> Mapping[str, object]: + """The request params this deployment forwards under drop_params, malformed values removed.""" + well_formed: Final = ( + {name: value for name, value in params.items() if _is_well_formed(name, value, params)} + if self.custom_llm_provider in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES + else params + ) + without_reasoning: Final = {name: value for name, value in well_formed.items() if name != "reasoning"} + reasoning: Final = well_formed.get("reasoning") + refusal: Final = self._reasoning_refusal(reasoning, model, declared_reasoning_efforts) + if isinstance(refusal, _ReasoningUnsupported): + return without_reasoning + reasoning_fields: Final = as_str_mapping(reasoning) + if reasoning_fields is None: + return well_formed + effort_kept: Final = not isinstance(refusal, _EffortDisabled) + kept_reasoning: Final = self._accepted_reasoning( + {key: value for key, value in reasoning_fields.items() if effort_kept or key != "effort"}, model + ) + return ( + {**without_reasoning, "reasoning": kept_reasoning} + if kept_reasoning or not reasoning_fields + else without_reasoning + ) + @staticmethod - def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": + def _enforce_min_max_output_tokens(max_output_tokens: object) -> object: """Raise sub-minimum max_output_tokens up to the OpenAI Responses API minimum. OpenAI's Responses API rejects max_output_tokens below 16 for every model @@ -413,19 +508,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): 'none' on models that support it). Apply the same validation used by the chat completions path. """ - params: Final = dict(response_api_optional_params) - - if "max_output_tokens" in params: - params["max_output_tokens"] = self._enforce_min_max_output_tokens(params.get("max_output_tokens")) - - if ( - self.custom_llm_provider == LlmProviders.OPENAI - and self._requests_reasoning_effort(params.get("reasoning")) - and not self._supports_reasoning_param(model=model) - ): - if drop_params or litellm.drop_params: - params.pop("reasoning", None) - else: + requested: Final[Mapping[str, object]] = response_api_optional_params + drops: Final = drop_params or litellm.drop_params + refusal: Final = ( + None if drops else self._reasoning_refusal(requested.get("reasoning"), model, declared_reasoning_efforts) + ) + match refusal: + case _ReasoningUnsupported(): raise litellm.UnsupportedParamsError( message=( f"{model} doesn't support `reasoning.effort` " @@ -434,28 +523,24 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ), status_code=400, ) + case _EffortDisabled(effort=disabled_effort): + raise litellm.UnsupportedParamsError( + message=unsupported_reasoning_effort_message("reasoning.effort", disabled_effort, model), + status_code=400, + ) + case None: + pass + params: Final = dict( + self.params_dropped_for_deployment(requested, model, declared_reasoning_efforts=declared_reasoning_efforts) + if drops + else requested + ) + + if "max_output_tokens" in params: + params["max_output_tokens"] = self._enforce_min_max_output_tokens(params.get("max_output_tokens")) lookup_name: Final = self._model_map_lookup_name(model) if self._is_gpt_5_model(model=lookup_name): - reasoning: Final = params.get("reasoning") or {} - requested_effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None - if ( - isinstance(reasoning, dict) - and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts - and self._effort_level_is_disabled(lookup_name, requested_effort) - ): - if not (drop_params or litellm.drop_params): - raise litellm.UnsupportedParamsError( - message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), - status_code=400, - ) - remaining: Final = reasoning.copy() - remaining.pop("effort", None) - if remaining: - params["reasoning"] = remaining - else: - params.pop("reasoning", None) forwarded_reasoning: Final = params.get("reasoning") effort: Final = forwarded_reasoning.get("effort") if isinstance(forwarded_reasoning, dict) else None supports_none: Final = self._supports_reasoning_effort_none(model=lookup_name) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b57cb1e4354 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1425,12 +1425,9 @@ def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | No return None if values is None else frozenset(values) -def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]: - model_info: Final = _openai_model_info(model) +def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Mapping[str, frozenset[str | None] | None]: if model_info is None: return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) - if not model_info.get("supports_reasoning"): - return MappingProxyType({}) return MappingProxyType( { "summary": _SUMMARY_VALUES, @@ -1440,6 +1437,13 @@ def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | N ) +def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]: + model_info: Final = _openai_model_info(model) + if model_info is not None and not model_info.get("supports_reasoning"): + return MappingProxyType({}) + return _openai_declared_reasoning_field_values(model_info) + + def _openai_accepts_reasoning_field( key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None] ) -> bool: @@ -1451,13 +1455,24 @@ def _openai_accepts_reasoning_field( return allowed is None or (isinstance(value, str | None) and value in allowed) -def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - values_by_field: Final = _openai_reasoning_field_values(model) +def _reasoning_with_accepted_fields( + reasoning: Mapping[str, object], values_by_field: Mapping[str, frozenset[str | None] | None] +) -> Mapping[str, object]: return { key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field) } +def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: + return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model)) + + +def openai_well_formed_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: + return _reasoning_with_accepted_fields( + reasoning, _openai_declared_reasoning_field_values(_openai_model_info(model)) + ) + + def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: if additional_drop_params.__class__ is not list: return () diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..6b298ed319b 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -642,3 +642,28 @@ async def test_native_count_answers_openais_rejection_with_its_status_and_text( assert result is not None assert (result.error, result.status_code, result.error_message) == (True, 400, rejection) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("declared", "effort_counted"), [(True, True), (False, False)], ids=["declared", "undeclared"]) +async def test_the_count_reads_the_deployments_declared_efforts_under_drop_params( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, declared: bool, effort_counted: bool +) -> None: + row: Final = litellm.model_cost["gpt-6.1-sol"] + effort: Final = next( + level for level in ("none", "minimal", "low") if row.get(f"supports_{level}_reasoning_effort") is False + ) + model_info: Final = {"reasoning_effort_levels": [effort]} if declared else {} + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + await OpenAITokenCounter().count_responses_request( + model_to_use="gpt-6.1-sol", + request={"model": "my-gpt", "input": "hi", "reasoning": {"effort": effort}}, + policy=DropPolicy(drop_params=True, additional_drop_params=frozenset()), + deployment={"litellm_params": {"api_key": "sk-test"}, "model_info": model_info}, + request_model="my-gpt", + ) + sent: Final = json.loads(route.calls.last.request.content) + + assert ("reasoning" in sent) is effort_counted diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 575183772f9..41e0c67348a 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -2,10 +2,12 @@ from typing import Final import pytest +import litellm from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, native_count_request, @@ -630,7 +632,7 @@ def test_a_saved_conversation_with_a_null_created_by_is_counted_without_it() -> policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) - body: Final = native_count_request(request, "gpt-5.6", policy) + body: Final = native_count_request(request, "gpt-5.6", policy, declared_reasoning_efforts=()) assert body["input"] == [compaction, tool_output, sent_created_by] @@ -641,7 +643,7 @@ def test_a_set_created_by_is_counted_without_it_under_drop_params() -> None: request: Final = {"model": "gpt-5.6", "input": [{**tool_output, "created_by": "user_1"}]} policy: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) - body: Final = native_count_request(request, "gpt-5.6", policy) + body: Final = native_count_request(request, "gpt-5.6", policy, declared_reasoning_efforts=()) assert body["input"] == [tool_output] @@ -649,7 +651,9 @@ def test_a_set_created_by_is_counted_without_it_under_drop_params() -> None: def test_a_text_input_is_counted_as_sent() -> None: policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) - body: Final = native_count_request({"model": "gpt-5.6", "input": "Hello"}, "gpt-5.6", policy) + body: Final = native_count_request( + {"model": "gpt-5.6", "input": "Hello"}, "gpt-5.6", policy, declared_reasoning_efforts=() + ) assert body["input"] == "Hello" @@ -662,6 +666,48 @@ def test_a_reasoning_item_is_counted_without_its_null_fields() -> None: } policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) - body: Final = native_count_request(request, "gpt-5.6", policy) + body: Final = native_count_request(request, "gpt-5.6", policy, declared_reasoning_efforts=()) assert body["input"] == [reasoning] + + +def _disabled_effort(model: str) -> str: + row: Final = litellm.model_cost[model] + return next( + level + for level in ("none", "minimal", "low", "medium", "high", "xhigh") + if row.get(f"supports_{level}_reasoning_effort") is False + ) + + +_DROP: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) + + +def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost_map) -> None: + malformed: Final = { + "tool_choice": "required", + "parallel_tool_calls": "true", + "previous_response_id": 12, + "reasoning": {"effort": _disabled_effort("gpt-6.1-sol"), "mode": "turbo", "summary": "auto"}, + } + + body: Final = native_count_request( + {"model": "gpt-6.1-sol", "input": "hi", **malformed}, "gpt-6.1-sol", _DROP, declared_reasoning_efforts=() + ) + inference: Final = OpenAIResponsesAPIConfig().map_openai_params(dict(malformed), "gpt-6.1-sol", True) + + assert body == {"model": "gpt-6.1-sol", "input": "hi", **inference} + assert inference == {"reasoning": {"summary": "auto"}} + + +def test_the_count_keeps_an_effort_the_deployment_declares(local_model_cost_map) -> None: + effort: Final = _disabled_effort("gpt-6.1-sol") + + body: Final = native_count_request( + {"model": "gpt-6.1-sol", "input": "hi", "reasoning": {"effort": effort}}, + "gpt-6.1-sol", + _DROP, + declared_reasoning_efforts=(effort,), + ) + + assert body["reasoning"] == {"effort": effort} diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 48bb0955f34..a720d210e75 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -12,6 +12,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -3201,6 +3202,100 @@ class TestReasoningFollowsModelSupport: assert mapped["reasoning"] == {"effort": "medium"} +_WEATHER_TOOL: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + + +class TestMalformedParamsFollowDropParams: + @pytest.mark.parametrize( + "config", + [OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig(), AzureOpenAIOSeriesResponsesAPIConfig()], + ids=["openai", "azure", "azure-o-series"], + ) + @pytest.mark.parametrize( + "params", + [ + pytest.param({"tool_choice": "required"}, id="required-without-tools"), + pytest.param({"tool_choice": "required", "tools": []}, id="required-with-empty-tools"), + pytest.param({"tool_choice": {"type": "function", "name": "get_weather"}}, id="function-without-tools"), + pytest.param({"parallel_tool_calls": "true"}, id="parallel-tool-calls-string"), + pytest.param({"previous_response_id": 12}, id="previous-response-id-int"), + pytest.param({"previous_response_id": ""}, id="previous-response-id-empty"), + ], + ) + def test_a_malformed_value_is_dropped_under_drop_params_and_sent_as_is_without( + self, local_model_cost_map, monkeypatch, config, params + ): + monkeypatch.setattr(litellm, "drop_params", False) + malformed: Final = next(name for name in params if name != "tools") + + dropped: Final = config.map_openai_params(dict(params), "gpt-6.1-sol", True) + sent: Final = config.map_openai_params(dict(params), "gpt-6.1-sol", False) + + assert malformed not in dropped + assert sent == params + + @pytest.mark.parametrize( + "params", + [ + pytest.param({"tool_choice": "auto"}, id="auto-without-tools"), + pytest.param({"tool_choice": "none"}, id="none-without-tools"), + pytest.param({"tool_choice": "required", "tools": [_WEATHER_TOOL]}, id="required-with-tools"), + pytest.param( + {"tool_choice": {"type": "function", "name": "get_weather"}, "tools": [_WEATHER_TOOL]}, + id="function-with-tools", + ), + pytest.param({"parallel_tool_calls": False}, id="parallel-tool-calls-bool"), + pytest.param({"parallel_tool_calls": None}, id="parallel-tool-calls-null"), + pytest.param({"previous_response_id": "resp_abc"}, id="previous-response-id-string"), + pytest.param({"tool_choice": "required", "prompt": {"id": "pmpt_abc"}}, id="required-with-stored-prompt"), + ], + ) + def test_a_well_formed_value_is_kept_under_drop_params(self, local_model_cost_map, params): + assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params + + def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map): + params: Final = {"tool_choice": "required", "parallel_tool_calls": "true", "previous_response_id": 12} + + assert HostedVLLMResponsesAPIConfig().map_openai_params(dict(params), "my-vllm-model", True) == params + + @pytest.mark.parametrize( + ("field", "value"), + [("mode", "turbo"), ("context", "forever"), ("summary", "sometimes"), ("generate_summary", "auto")], + ) + def test_a_reasoning_value_the_model_row_does_not_declare_is_dropped_under_drop_params( + self, local_model_cost_map, monkeypatch, field, value + ): + monkeypatch.setattr(litellm, "drop_params", False) + row: Final = litellm.model_cost["gpt-6.1-sol"] + declared: Final = { + "effort": "high", + "mode": row["reasoning_modes"][0], + "context": row["reasoning_contexts"][0], + "summary": "auto", + } + reasoning: Final = {**declared, field: value} + + dropped: Final = OpenAIResponsesAPIConfig().map_openai_params({"reasoning": reasoning}, "gpt-6.1-sol", True) + sent: Final = OpenAIResponsesAPIConfig().map_openai_params({"reasoning": reasoning}, "gpt-6.1-sol", False) + + assert dropped["reasoning"] == {key: kept for key, kept in declared.items() if key != field} + assert sent["reasoning"] == reasoning + + @pytest.mark.parametrize( + ("config", "model"), + [(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (OpenAIResponsesAPIConfig(), "my-fine-tuned-model")], + ids=["azure", "outside-openai-catalog"], + ) + def test_reasoning_fields_pass_through_where_the_openai_catalog_does_not_apply( + self, local_model_cost_map, config, model + ): + reasoning: Final = {"effort": "high", "mode": "turbo", "generate_summary": "auto"} + + mapped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True) + + assert mapped["reasoning"] == reasoning + + @pytest.mark.asyncio async def test_openai_responses_litellm_router_no_metadata(): """ From e2e7153896678f7eacc027adcb1cfa3c4748fb69 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:16:41 -0700 Subject: [PATCH 2/9] fix(responses): drop undeclared reasoning values and unknown efforts, and round count-route efforts like inference A row with no declared reasoning modes or contexts is checked against what bundled OpenAI rows declare, a non-reasoning model keeps only summary, an effort outside the known levels drops unless the deployment declares it, Azure o-series runs only the malformed-value checks, and the count route rounds the effort to the deployment's declared levels --- .../responses/o_series_transformation.py | 4 +- .../responses/count_tokens/token_counter.py | 31 ++++++-- .../llms/openai/responses/transformation.py | 32 +++++--- litellm/responses/utils.py | 50 ++++++++++--- .../count_tokens/test_token_counter.py | 21 ++++++ .../test_openai_responses_transformation.py | 75 +++++++++++++++++++ tests/unit/responses/test_responses_utils.py | 31 +++++++- 7 files changed, 215 insertions(+), 29 deletions(-) diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 259a94bd14d..59abafea3f8 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -65,9 +65,7 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): """ if not drop_params: return dict(response_api_optional_params) - kept: Final = self.params_dropped_for_deployment( - response_api_optional_params, model, declared_reasoning_efforts=() - ) + kept: Final = self._well_formed_params(response_api_optional_params) if "temperature" in kept: verbose_logger.debug( "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 88ce5002803..bef80dd9612 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -35,7 +35,10 @@ from litellm.llms.openai.workload_identity import ( openai_static_api_key, resolve_openai_workload_identity_config, ) -from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels +from litellm.router_utils.reasoning_effort_capability import ( + deployment_reasoning_effort_levels, + resolve_deployment_reasoning_effort, +) from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests @@ -96,6 +99,24 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +def _with_deployment_reasoning( + request: Mapping[str, object], model: str, model_info: object, policy: DropPolicy +) -> Mapping[str, object]: + reasoning: Final = as_str_mapping(request.get("reasoning")) + if reasoning is None: + return request + resolved: Final = resolve_deployment_reasoning_effort( + reasoning, + model=model, + custom_llm_provider=LlmProviders.OPENAI.value, + deployment_model_info=model_info, + model_group=None, + drop_params=policy.drop_params, + ).value + without_reasoning: Final = {field: value for field, value in request.items() if field != "reasoning"} + return without_reasoning if resolved is None else {**without_reasoning, "reasoning": resolved} + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -219,14 +240,12 @@ class OpenAITokenCounter(BaseTokenCounter): resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) + model_info: Final = None if deployment is None else deployment.get("model_info") body: Final = native_count_request( - request, + _with_deployment_reasoning(request, model_to_use, model_info, policy), model_to_use, policy, - declared_reasoning_efforts=deployment_reasoning_effort_levels( - None if deployment is None else deployment.get("model_info") - ) - or (), + declared_reasoning_efforts=deployment_reasoning_effort_levels(model_info) or (), ) return await self._counted( resolved, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index c628d02a070..2a77291e6dc 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -2,7 +2,7 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_type_hints +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_args, get_type_hints import httpx from openai.types.responses import ResponseReasoningItem @@ -50,6 +50,7 @@ from litellm.responses.utils import openai_well_formed_reasoning from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * +from litellm.types.llms.openai import REASONING_EFFORT from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -321,6 +322,7 @@ _TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"}) _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE) +_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: @@ -499,19 +501,27 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return None def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - if self.custom_llm_provider == LlmProviders.OPENAI and OpenAIGPTConfig.is_openai_catalog_model(model): - return openai_well_formed_reasoning(reasoning, model) - return reasoning + if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model): + return reasoning + return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model)) + + def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool: + if effort is None or self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: + return True + return isinstance(effort, str) and ( + effort in _REASONING_EFFORTS or effort in (declared_reasoning_efforts or ()) + ) + + def _well_formed_params(self, params: Mapping[str, object]) -> Mapping[str, object]: + if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: + return params + return {name: value for name, value in params.items() if _is_well_formed(name, value, params)} def params_dropped_for_deployment( self, params: Mapping[str, object], model: str, *, declared_reasoning_efforts: Collection[str] | None ) -> Mapping[str, object]: """The request params this deployment forwards under drop_params, malformed values removed.""" - well_formed: Final = ( - {name: value for name, value in params.items() if _is_well_formed(name, value, params)} - if self.custom_llm_provider in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES - else params - ) + well_formed: Final = self._well_formed_params(params) without_reasoning: Final = {name: value for name, value in well_formed.items() if name != "reasoning"} reasoning: Final = well_formed.get("reasoning") refusal: Final = self._reasoning_refusal(reasoning, model, declared_reasoning_efforts) @@ -520,7 +530,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): reasoning_fields: Final = as_str_mapping(reasoning) if reasoning_fields is None: return well_formed - effort_kept: Final = not isinstance(refusal, _EffortDisabled) + effort_kept: Final = not isinstance(refusal, _EffortDisabled) and self._is_well_formed_effort( + reasoning_fields.get("effort"), declared_reasoning_efforts + ) kept_reasoning: Final = self._accepted_reasoning( {key: value for key, value in reasoning_fields.items() if effort_kept or key != "effort"}, model ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 2cb6fc8b436..73933901000 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -2,11 +2,12 @@ import base64 import os import re from collections.abc import Callable, Collection, Iterable, Mapping, Sequence -from functools import reduce +from functools import lru_cache, reduce +from itertools import chain from types import MappingProxyType from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion @@ -14,6 +15,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -1412,6 +1414,7 @@ def reasoning_with_summary_default( _SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) +_NON_REASONING_MODEL_FIELD_VALUES: Final = MappingProxyType({"summary": _SUMMARY_VALUES}) def _openai_model_info(model: str) -> ModelInfo | None: @@ -1421,18 +1424,42 @@ def _openai_model_info(model: str) -> ModelInfo | None: return None if isinstance(model_info, ModelInfoLookupMiss) else model_info -def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | None: - return None if values is None else frozenset(values) +class _ReasoningDeclarations(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_provider: str | None = None + reasoning_modes: tuple[str, ...] | None = None + reasoning_contexts: tuple[str, ...] | None = None + + +_BUNDLED_REASONING_DECLARATIONS: Final = TypeAdapter(dict[str, _ReasoningDeclarations]) + + +@lru_cache(maxsize=1) +def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]: + bundled: Final = _BUNDLED_REASONING_DECLARATIONS.validate_json(GetModelCostMap.read_local_model_cost_map_text()) + openai_rows: Final = tuple(row for row in bundled.values() if row.litellm_provider == "openai") + return MappingProxyType( + { + "mode": frozenset(chain.from_iterable(row.reasoning_modes or () for row in openai_rows)), + "context": frozenset(chain.from_iterable(row.reasoning_contexts or () for row in openai_rows)), + } + ) + + +def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) -> frozenset[str | None]: + return undeclared if values is None else frozenset(values) def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Mapping[str, frozenset[str | None] | None]: if model_info is None: return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) + declared_anywhere: Final = _reasoning_values_any_openai_row_declares() return MappingProxyType( { "summary": _SUMMARY_VALUES, - "mode": _declared_values(model_info.get("reasoning_modes")), - "context": _declared_values(model_info.get("reasoning_contexts")), + "mode": _declared_values(model_info.get("reasoning_modes"), declared_anywhere["mode"]), + "context": _declared_values(model_info.get("reasoning_contexts"), declared_anywhere["context"]), } ) @@ -1467,10 +1494,15 @@ def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> M return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model)) -def openai_well_formed_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - return _reasoning_with_accepted_fields( - reasoning, _openai_declared_reasoning_field_values(_openai_model_info(model)) +def openai_well_formed_reasoning( + reasoning: Mapping[str, object], model: str, *, model_reasons: bool +) -> Mapping[str, object]: + values_by_field: Final = ( + _openai_declared_reasoning_field_values(_openai_model_info(model)) + if model_reasons + else _NON_REASONING_MODEL_FIELD_VALUES ) + return _reasoning_with_accepted_fields(reasoning, values_by_field) def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 6b298ed319b..7fb7f0cbded 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -667,3 +667,24 @@ async def test_the_count_reads_the_deployments_declared_efforts_under_drop_param sent: Final = json.loads(route.calls.last.request.content) assert ("reasoning" in sent) is effort_counted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "send-as-is"]) +async def test_the_count_rounds_the_effort_to_a_declared_level_as_inference_does( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, drop_params: bool +) -> None: + declared: Final = ["low", "medium", "high"] + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + await OpenAITokenCounter().count_responses_request( + model_to_use="gpt-6.1-sol", + request={"model": "my-gpt", "input": "hi", "reasoning": {"effort": "max", "summary": "auto"}}, + policy=DropPolicy(drop_params=drop_params, additional_drop_params=frozenset()), + deployment={"litellm_params": {"api_key": "sk-test"}, "model_info": {"reasoning_effort_levels": declared}}, + request_model="my-gpt", + ) + sent: Final = json.loads(route.calls.last.request.content) + + assert sent["reasoning"] == {"effort": "high", "summary": "auto"} diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 70213534caa..a4a930c42a2 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -18,6 +18,7 @@ from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIC from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig @@ -3336,6 +3337,18 @@ class TestMalformedParamsFollowDropParams: assert malformed not in dropped assert sent == params + def test_azure_o_series_forwards_a_row_disabled_effort_while_dropping_malformed_values( + self, local_model_cost_map, monkeypatch + ): + row: Final = litellm.model_cost["gpt-6.1-sol"] + monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**row, "supports_minimal_reasoning_effort": False}) + + mapped: Final = AzureOpenAIOSeriesResponsesAPIConfig().map_openai_params( + {"reasoning": {"effort": "minimal"}, "tool_choice": "required"}, "gpt-6.1-sol", True + ) + + assert mapped == {"reasoning": {"effort": "minimal"}} + @pytest.mark.parametrize( "params", [ @@ -3383,6 +3396,68 @@ class TestMalformedParamsFollowDropParams: assert dropped["reasoning"] == {key: kept for key, kept in declared.items() if key != field} assert sent["reasoning"] == reasoning + @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) + def test_a_row_without_reasoning_declarations_drops_values_no_openai_row_declares( + self, local_model_cost_map, monkeypatch, field, declarations + ): + monkeypatch.setattr(litellm, "drop_params", False) + undeclared_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" and row.get("supports_reasoning") and declarations not in row + ) + declared_elsewhere: Final = litellm.model_cost["gpt-6.1-sol"][declarations][0] + malformed: Final = {"summary": "auto", field: "not-a-declared-value"} + declared: Final = {"summary": "auto", field: declared_elsewhere} + config: Final = OpenAIResponsesAPIConfig() + + dropped: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, True) + sent: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, False) + kept: Final = config.map_openai_params({"reasoning": dict(declared)}, undeclared_model, True) + + assert dropped["reasoning"] == {"summary": "auto"} + assert sent["reasoning"] == malformed + assert kept["reasoning"] == declared + + def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params(self, local_model_cost_map, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + non_reasoning_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" + and not row.get("supports_reasoning") + and OpenAIGPTConfig.is_openai_catalog_model(name) + ) + request: Final = {"reasoning": {"summary": "auto", "mode": "standard", "context": "auto"}, "tool_choice": "auto"} + config: Final = OpenAIResponsesAPIConfig() + + dropped: Final = config.map_openai_params(dict(request), non_reasoning_model, True) + sent: Final = config.map_openai_params(dict(request), non_reasoning_model, False) + + assert dropped == {"reasoning": {"summary": "auto"}, "tool_choice": "auto"} + assert sent == request + + @pytest.mark.parametrize( + ("config", "model"), + [(OpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol")], + ids=["openai", "azure"], + ) + def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it( + self, local_model_cost_map, monkeypatch, config, model + ): + monkeypatch.setattr(litellm, "drop_params", False) + reasoning: Final = {"effort": "not-an-effort", "summary": "auto"} + + dropped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True) + sent: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, False) + declared: Final = config.map_openai_params_for_deployment( + {"reasoning": dict(reasoning)}, model, True, declared_reasoning_efforts=("not-an-effort",) + ) + + assert dropped["reasoning"] == {"summary": "auto"} + assert sent["reasoning"] == reasoning + assert declared["reasoning"] == reasoning + @pytest.mark.parametrize( ("config", "model"), [(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (OpenAIResponsesAPIConfig(), "my-fine-tuned-model")], diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 0863abd1f1e..5e6c98267fb 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -10,7 +10,8 @@ import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig -from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils, openai_bridge_reasoning from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.utils import ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map @@ -1009,3 +1010,31 @@ def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): else: litellm.model_cost[model] = previous_row _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) +def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_drop_params( + local_model_cost_map, field, declarations +): + undeclared_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" + and row.get("supports_reasoning") + and declarations not in row + and OpenAIGPTConfig.is_openai_catalog_model(name) + ) + requested: Final = {"effort": "high", "summary": "auto", field: "not-a-declared-value"} + + def bridged(drop_params: bool) -> object: + return openai_bridge_reasoning( + requested=requested, + effort_kept=True, + summary_alias=None, + model=undeclared_model, + drop_params=drop_params, + additional_drop_params=None, + ) + + assert bridged(True) == {"effort": "high", "summary": "auto"} + assert bridged(False) == requested From db720da71fab2837e9e800fe72f19f750b725772 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:42:28 -0700 Subject: [PATCH 3/9] fix(responses): drop a tool_choice naming a tool the request does not send, and run the unknown-effort drop on Azure o-series --- .../responses/o_series_transformation.py | 16 +++- .../llms/openai/responses/transformation.py | 40 +++++++++- .../test_openai_responses_transformation.py | 76 ++++++++++++++++++- tests/unit/responses/test_responses_utils.py | 21 +++++ 4 files changed, 145 insertions(+), 8 deletions(-) diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 59abafea3f8..cf33d623396 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -8,6 +8,7 @@ Translations handled by LiteLLM: - Other parameters follow base Azure OpenAI Responses API behavior """ +from collections.abc import Collection, Mapping from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger @@ -51,12 +52,14 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): return o_series_supported_params - def map_openai_params( + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - ) -> dict: + *, + declared_reasoning_efforts: Collection[str] | None, + ) -> Mapping[str, object]: """ Map OpenAI parameters for Azure OpenAI O-series Responses API. @@ -65,13 +68,20 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): """ if not drop_params: return dict(response_api_optional_params) - kept: Final = self._well_formed_params(response_api_optional_params) + kept: Final = self.params_dropped_for_deployment( + response_api_optional_params, model, declared_reasoning_efforts=declared_reasoning_efforts + ) if "temperature" in kept: verbose_logger.debug( "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model ) return {name: value for name, value in kept.items() if name != "temperature"} + def _reasoning_refusal( + self, reasoning: object, model: str, declared_reasoning_efforts: Collection[str] | None + ) -> None: + return None + def is_o_series_model(self, model: str) -> bool: """ Check if the model is an O-series model. diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 2a77291e6dc..f3266383c91 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,6 +1,7 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_args, get_type_hints @@ -325,13 +326,50 @@ _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProvid _REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) +_NON_HOSTED_TOOL_TYPES: Final = frozenset({"function", "custom", "mcp", "namespace"}) + + +def _tool_mappings(tools: object) -> tuple[Mapping[str, object], ...]: + return tuple(mapping for mapping in map(as_str_mapping, _json_array_or_empty(tools)) if mapping is not None) + + +def _sent_tools(tools: object) -> tuple[Mapping[str, object], ...]: + top_level: Final = _tool_mappings(tools) + namespaced: Final = (_tool_mappings(tool.get("tools")) for tool in top_level if tool.get("type") == "namespace") + return top_level + tuple(chain.from_iterable(namespaced)) + + +def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], ...]) -> bool: + choice: Final = as_str_mapping(tool_choice) + if choice is None: + return True + match choice.get("type"): + case "allowed_tools": + allowed: Final = choice.get("tools") + entries: Final = _json_array_or_empty(allowed) + return not isinstance(allowed, list | tuple) or all(_names_a_sent_tool(entry, tools) for entry in entries) + case "function": + return any(tool.get("type") == "function" and tool.get("name") == choice.get("name") for tool in tools) + case "custom": + return any(tool.get("type") == "custom" for tool in tools) + case "mcp": + return any( + tool.get("type") == "mcp" and tool.get("server_label") == choice.get("server_label") for tool in tools + ) + case str(): + return any(tool.get("type") not in _NON_HOSTED_TOOL_TYPES for tool in tools) + case _: + return True + + def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: if tool_choice is None or params.get("prompt") is not None: return True tools: Final = params.get("tools") if isinstance(tool_choice, str): return tool_choice in _TOOL_CHOICES_WITHOUT_TOOLS or (tool_choice == "required" and bool(tools)) - return bool(tools) + listed: Final = _json_array_or_empty(tools) + return bool(tools) and (not isinstance(tools, list | tuple) or _names_a_sent_tool(tool_choice, _sent_tools(listed))) def _is_well_formed(name: str, value: object, params: Mapping[str, object]) -> bool: diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index a4a930c42a2..53d40bb9e01 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3306,6 +3306,9 @@ class TestReasoningFollowsModelSupport: _WEATHER_TOOL: Final = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} +_SQL_TOOL: Final = {"type": "custom", "name": "run_sql"} +_DOCS_MCP_TOOL: Final = {"type": "mcp", "server_label": "docs", "server_url": "https://example.com/mcp"} +_WEATHER_NAMESPACE: Final = {"type": "namespace", "name": "weather", "description": "Weather", "tools": [_WEATHER_TOOL]} class TestMalformedParamsFollowDropParams: @@ -3320,6 +3323,24 @@ class TestMalformedParamsFollowDropParams: pytest.param({"tool_choice": "required"}, id="required-without-tools"), pytest.param({"tool_choice": "required", "tools": []}, id="required-with-empty-tools"), pytest.param({"tool_choice": {"type": "function", "name": "get_weather"}}, id="function-without-tools"), + pytest.param( + {"tool_choice": {"type": "function", "name": "get_time"}, "tools": [_WEATHER_TOOL]}, + id="function-naming-a-tool-not-sent", + ), + pytest.param({"tool_choice": {"type": "custom", "name": "run_sql"}, "tools": [_WEATHER_TOOL]}, id="custom"), + pytest.param({"tool_choice": {"type": "mcp", "server_label": "wiki"}, "tools": [_DOCS_MCP_TOOL]}, id="mcp"), + pytest.param({"tool_choice": {"type": "file_search"}, "tools": [_WEATHER_TOOL]}, id="hosted-tool-not-sent"), + pytest.param( + { + "tool_choice": { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_time"}], + }, + "tools": [_WEATHER_TOOL], + }, + id="allowed-tools-naming-a-tool-not-sent", + ), pytest.param({"parallel_tool_calls": "true"}, id="parallel-tool-calls-string"), pytest.param({"previous_response_id": 12}, id="previous-response-id-int"), pytest.param({"previous_response_id": ""}, id="previous-response-id-empty"), @@ -3363,11 +3384,50 @@ class TestMalformedParamsFollowDropParams: pytest.param({"parallel_tool_calls": None}, id="parallel-tool-calls-null"), pytest.param({"previous_response_id": "resp_abc"}, id="previous-response-id-string"), pytest.param({"tool_choice": "required", "prompt": {"id": "pmpt_abc"}}, id="required-with-stored-prompt"), + pytest.param( + {"tool_choice": {"type": "function", "name": "get_weather"}, "tools": [_WEATHER_NAMESPACE]}, + id="function-in-a-namespace", + ), + pytest.param( + {"tool_choice": {"type": "custom", "name": "other"}, "tools": [_SQL_TOOL]}, + id="custom-with-a-custom-tool", + ), + pytest.param( + {"tool_choice": {"type": "mcp", "server_label": "docs"}, "tools": [_DOCS_MCP_TOOL]}, id="mcp-label-sent" + ), + pytest.param( + {"tool_choice": {"type": "computer_use"}, "tools": [{"type": "computer"}]}, + id="hosted-alias-of-a-sent-tool", + ), + pytest.param( + { + "tool_choice": { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_weather"}, {"type": "mcp", "server_label": "docs"}], + }, + "tools": [_WEATHER_TOOL, _DOCS_MCP_TOOL], + }, + id="allowed-tools-all-sent", + ), ], ) def test_a_well_formed_value_is_kept_under_drop_params(self, local_model_cost_map, params): assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params + @pytest.mark.parametrize( + "params", + [ + pytest.param({"tool_choice": {"name": "get_time"}, "tools": [_WEATHER_TOOL]}, id="choice-without-a-type"), + pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="choice-not-an-object"), + pytest.param( + {"tool_choice": {"type": "function", "name": "get_time"}, "tools": _WEATHER_TOOL}, id="tools-not-a-list" + ), + ], + ) + def test_a_tool_choice_the_check_cannot_read_is_kept_under_drop_params(self, local_model_cost_map, params): + assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params + def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map): params: Final = {"tool_choice": "required", "parallel_tool_calls": "true", "previous_response_id": 12} @@ -3439,8 +3499,12 @@ class TestMalformedParamsFollowDropParams: @pytest.mark.parametrize( ("config", "model"), - [(OpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol")], - ids=["openai", "azure"], + [ + (OpenAIResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureOpenAIOSeriesResponsesAPIConfig(), "o_series/gpt-6.1-sol"), + ], + ids=["openai", "azure", "azure-o-series"], ) def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it( self, local_model_cost_map, monkeypatch, config, model @@ -3450,8 +3514,12 @@ class TestMalformedParamsFollowDropParams: dropped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True) sent: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, False) - declared: Final = config.map_openai_params_for_deployment( - {"reasoning": dict(reasoning)}, model, True, declared_reasoning_efforts=("not-an-effort",) + declared: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=config, + response_api_optional_params={"reasoning": dict(reasoning)}, + drop_params=True, + declared_reasoning_efforts=("not-an-effort",), ) assert dropped["reasoning"] == {"summary": "auto"} diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 5e6c98267fb..8de72b4c3bb 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1038,3 +1038,24 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr assert bridged(True) == {"effort": "high", "summary": "auto"} assert bridged(False) == requested + +def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map): + non_reasoning_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" + and row.get("mode") == "chat" + and row.get("supports_reasoning") is False + and OpenAIGPTConfig.is_openai_catalog_model(name) + ) + + bridged: Final = openai_bridge_reasoning( + requested={"effort": "low", "summary": "auto", "mode": "standard"}, + effort_kept=True, + summary_alias=None, + model=non_reasoning_model, + drop_params=True, + additional_drop_params=None, + ) + + assert bridged == {"effort": "low"} From e616590e5832538fafbfcf4e0dd8047c4c1e1022 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:22:25 -0700 Subject: [PATCH 4/9] fix(responses): drop Azure and off-catalog reasoning values and namespace tool_choice under drop_params Azure and models missing from the cost map now check reasoning mode, context and summary against the values any OpenAI row declares and drop unknown keys. A function tool_choice must name a top-level function, since OpenAI rejects one naming a function inside a namespace. --- .../llms/openai/responses/transformation.py | 17 +++---- litellm/responses/utils.py | 19 ++++--- .../test_openai_responses_transformation.py | 51 ++++++++++++++----- tests/unit/responses/test_responses_utils.py | 20 ++++++++ 4 files changed, 75 insertions(+), 32 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index f3266383c91..e3b1973f505 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,7 +1,6 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache -from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_args, get_type_hints @@ -47,7 +46,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( tools_with_anthropic_tool_search_mapped, tools_without_anthropic_tool_search, ) -from litellm.responses.utils import openai_well_formed_reasoning +from litellm.responses.utils import openai_api_well_formed_reasoning, openai_well_formed_reasoning from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * @@ -333,12 +332,6 @@ def _tool_mappings(tools: object) -> tuple[Mapping[str, object], ...]: return tuple(mapping for mapping in map(as_str_mapping, _json_array_or_empty(tools)) if mapping is not None) -def _sent_tools(tools: object) -> tuple[Mapping[str, object], ...]: - top_level: Final = _tool_mappings(tools) - namespaced: Final = (_tool_mappings(tool.get("tools")) for tool in top_level if tool.get("type") == "namespace") - return top_level + tuple(chain.from_iterable(namespaced)) - - def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], ...]) -> bool: choice: Final = as_str_mapping(tool_choice) if choice is None: @@ -369,7 +362,9 @@ def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> b if isinstance(tool_choice, str): return tool_choice in _TOOL_CHOICES_WITHOUT_TOOLS or (tool_choice == "required" and bool(tools)) listed: Final = _json_array_or_empty(tools) - return bool(tools) and (not isinstance(tools, list | tuple) or _names_a_sent_tool(tool_choice, _sent_tools(listed))) + return bool(tools) and ( + not isinstance(tools, list | tuple) or _names_a_sent_tool(tool_choice, _tool_mappings(listed)) + ) def _is_well_formed(name: str, value: object, params: Mapping[str, object]) -> bool: @@ -539,8 +534,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return None def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model): + if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: return reasoning + if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model): + return openai_api_well_formed_reasoning(reasoning) return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model)) def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 73933901000..c0365e530ca 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -18,7 +18,6 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel -from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import ( AllMessageValues, InputTokensDetails, @@ -1452,14 +1451,14 @@ def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) - def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Mapping[str, frozenset[str | None] | None]: - if model_info is None: - return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) declared_anywhere: Final = _reasoning_values_any_openai_row_declares() + modes: Final = None if model_info is None else model_info.get("reasoning_modes") + contexts: Final = None if model_info is None else model_info.get("reasoning_contexts") return MappingProxyType( { "summary": _SUMMARY_VALUES, - "mode": _declared_values(model_info.get("reasoning_modes"), declared_anywhere["mode"]), - "context": _declared_values(model_info.get("reasoning_contexts"), declared_anywhere["context"]), + "mode": _declared_values(modes, declared_anywhere["mode"]), + "context": _declared_values(contexts, declared_anywhere["context"]), } ) @@ -1494,6 +1493,10 @@ def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> M return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model)) +def openai_api_well_formed_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object]: + return _reasoning_with_accepted_fields(reasoning, _openai_declared_reasoning_field_values(None)) + + def openai_well_formed_reasoning( reasoning: Mapping[str, object], model: str, *, model_reasons: bool ) -> Mapping[str, object]: @@ -1525,11 +1528,7 @@ def openai_bridge_reasoning( return None kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"} with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai") - accepted: Final = ( - _openai_accepted_reasoning(with_summary, model) - if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model) - else {**with_summary} - ) + accepted: Final = _openai_accepted_reasoning(with_summary, model) if drop_params else {**with_summary} remaining: Final = as_str_mapping( _apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort") ) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 53d40bb9e01..d693df4d72c 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3327,6 +3327,10 @@ class TestMalformedParamsFollowDropParams: {"tool_choice": {"type": "function", "name": "get_time"}, "tools": [_WEATHER_TOOL]}, id="function-naming-a-tool-not-sent", ), + pytest.param( + {"tool_choice": {"type": "function", "name": "get_weather"}, "tools": [_WEATHER_NAMESPACE]}, + id="function-inside-a-namespace", + ), pytest.param({"tool_choice": {"type": "custom", "name": "run_sql"}, "tools": [_WEATHER_TOOL]}, id="custom"), pytest.param({"tool_choice": {"type": "mcp", "server_label": "wiki"}, "tools": [_DOCS_MCP_TOOL]}, id="mcp"), pytest.param({"tool_choice": {"type": "file_search"}, "tools": [_WEATHER_TOOL]}, id="hosted-tool-not-sent"), @@ -3384,10 +3388,6 @@ class TestMalformedParamsFollowDropParams: pytest.param({"parallel_tool_calls": None}, id="parallel-tool-calls-null"), pytest.param({"previous_response_id": "resp_abc"}, id="previous-response-id-string"), pytest.param({"tool_choice": "required", "prompt": {"id": "pmpt_abc"}}, id="required-with-stored-prompt"), - pytest.param( - {"tool_choice": {"type": "function", "name": "get_weather"}, "tools": [_WEATHER_NAMESPACE]}, - id="function-in-a-namespace", - ), pytest.param( {"tool_choice": {"type": "custom", "name": "other"}, "tools": [_SQL_TOOL]}, id="custom-with-a-custom-tool", @@ -3429,7 +3429,12 @@ class TestMalformedParamsFollowDropParams: assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map): - params: Final = {"tool_choice": "required", "parallel_tool_calls": "true", "previous_response_id": 12} + params: Final = { + "tool_choice": "required", + "parallel_tool_calls": "true", + "previous_response_id": 12, + "reasoning": {"effort": "high", "mode": "turbo", "generate_summary": "auto"}, + } assert HostedVLLMResponsesAPIConfig().map_openai_params(dict(params), "my-vllm-model", True) == params @@ -3528,17 +3533,39 @@ class TestMalformedParamsFollowDropParams: @pytest.mark.parametrize( ("config", "model"), - [(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (OpenAIResponsesAPIConfig(), "my-fine-tuned-model")], - ids=["azure", "outside-openai-catalog"], + [ + (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureOpenAIOSeriesResponsesAPIConfig(), "o_series/gpt-6.1-sol"), + (OpenAIResponsesAPIConfig(), "my-fine-tuned-model"), + ], + ids=["azure", "azure-o-series", "outside-openai-catalog"], ) - def test_reasoning_fields_pass_through_where_the_openai_catalog_does_not_apply( - self, local_model_cost_map, config, model + def test_reasoning_values_no_openai_row_declares_are_dropped_where_the_openai_catalog_does_not_apply( + self, local_model_cost_map, monkeypatch, config, model ): - reasoning: Final = {"effort": "high", "mode": "turbo", "generate_summary": "auto"} + monkeypatch.setattr(litellm, "drop_params", False) + row: Final = litellm.model_cost["gpt-6.1-sol"] + declared: Final = { + "effort": "high", + "mode": row["reasoning_modes"][-1], + "context": row["reasoning_contexts"][-1], + "summary": "concise", + } + malformed: Final = { + "effort": "high", + "mode": "turbo", + "context": "forever", + "summary": "sometimes", + "generate_summary": "auto", + } - mapped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True) + dropped: Final = config.map_openai_params({"reasoning": dict(malformed)}, model, True) + kept: Final = config.map_openai_params({"reasoning": dict(declared)}, model, True) + sent: Final = config.map_openai_params({"reasoning": dict(malformed)}, model, False) - assert mapped["reasoning"] == reasoning + assert dropped["reasoning"] == {"effort": "high"} + assert kept["reasoning"] == declared + assert sent["reasoning"] == malformed @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 8de72b4c3bb..a9db0127ca6 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1039,6 +1039,26 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr assert bridged(True) == {"effort": "high", "summary": "auto"} assert bridged(False) == requested +def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_outside_the_openai_catalog( + local_model_cost_map, +): + declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] + requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "generate_summary": "auto"} + + def bridged(drop_params: bool) -> object: + return openai_bridge_reasoning( + requested=requested, + effort_kept=True, + summary_alias=None, + model="my-fine-tuned-model", + drop_params=drop_params, + additional_drop_params=None, + ) + + assert bridged(True) == {"effort": "high", "mode": declared_mode} + assert bridged(False) == requested + + def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map): non_reasoning_model: Final = next( name From 09676018655a24f895cf58fa631ddf0cbc9fc396 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:15:25 -0700 Subject: [PATCH 5/9] fix(responses): keep off-catalog reasoning keys, read reasoning lists from the loaded cost map, drop non-object reasoning and tool_choice --- .../llms/openai/responses/transformation.py | 10 ++- litellm/responses/utils.py | 83 ++++++++++++------- .../test_openai_responses_transformation.py | 39 +++++++-- tests/unit/responses/test_responses_utils.py | 6 +- 4 files changed, 99 insertions(+), 39 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index e3b1973f505..e6dd20ef15b 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -335,7 +335,7 @@ def _tool_mappings(tools: object) -> tuple[Mapping[str, object], ...]: def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], ...]) -> bool: choice: Final = as_str_mapping(tool_choice) if choice is None: - return True + return False match choice.get("type"): case "allowed_tools": allowed: Final = choice.get("tools") @@ -375,6 +375,8 @@ def _is_well_formed(name: str, value: object, params: Mapping[str, object]) -> b return value is None or isinstance(value, bool) case "previous_response_id": return value is None or (isinstance(value, str) and value != "") + case "reasoning": + return value is None or as_str_mapping(value) is not None case _: return True @@ -536,8 +538,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: return reasoning - if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model): - return openai_api_well_formed_reasoning(reasoning) + if self.custom_llm_provider != LlmProviders.OPENAI: + return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=False) + if not OpenAIGPTConfig.is_openai_catalog_model(model): + return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=True) return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model)) def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index c0365e530ca..e2540656197 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -7,7 +7,7 @@ from itertools import chain from types import MappingProxyType from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion @@ -15,7 +15,6 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( @@ -1423,29 +1422,46 @@ def _openai_model_info(model: str) -> ModelInfo | None: return None if isinstance(model_info, ModelInfoLookupMiss) else model_info -class _ReasoningDeclarations(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - litellm_provider: str | None = None - reasoning_modes: tuple[str, ...] | None = None - reasoning_contexts: tuple[str, ...] | None = None +def _declared_strings(rows: Iterable[Mapping[str, object]], field: str) -> frozenset[str]: + return frozenset(chain.from_iterable(_string_items(row.get(field)) for row in rows)) -_BUNDLED_REASONING_DECLARATIONS: Final = TypeAdapter(dict[str, _ReasoningDeclarations]) +def _string_items(values: object) -> tuple[str, ...]: + listed: Final = iter_items(values) if type(values) in (list, tuple) else () + return tuple(value for value in listed if isinstance(value, str)) + + +_NO_COST_MAP: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _model_cost() -> Mapping[str, object]: + model_cost: Final = as_str_mapping(attribute_or_none(litellm, "model_cost")) + return _NO_COST_MAP if model_cost is None else model_cost @lru_cache(maxsize=1) -def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]: - bundled: Final = _BUNDLED_REASONING_DECLARATIONS.validate_json(GetModelCostMap.read_local_model_cost_map_text()) - openai_rows: Final = tuple(row for row in bundled.values() if row.litellm_provider == "openai") +def _reasoning_values_declared_in_cost_map( + _cost_map_id: int, _cost_map_generation: int +) -> Mapping[str, frozenset[str]]: + openai_rows: Final = tuple( + row + for row in map(as_str_mapping, tuple(_model_cost().values())) + if row is not None and row.get("litellm_provider") == "openai" + ) return MappingProxyType( { - "mode": frozenset(chain.from_iterable(row.reasoning_modes or () for row in openai_rows)), - "context": frozenset(chain.from_iterable(row.reasoning_contexts or () for row in openai_rows)), + "mode": _declared_strings(openai_rows, "reasoning_modes"), + "context": _declared_strings(openai_rows, "reasoning_contexts"), } ) +def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]: + from litellm.utils import get_model_cost_mutation_generation + + return _reasoning_values_declared_in_cost_map(id(_model_cost()), get_model_cost_mutation_generation()) + + def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) -> frozenset[str | None]: return undeclared if values is None else frozenset(values) @@ -1463,38 +1479,49 @@ def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Map ) -def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]: - model_info: Final = _openai_model_info(model) - if model_info is not None and not model_info.get("supports_reasoning"): - return MappingProxyType({}) - return _openai_declared_reasoning_field_values(model_info) - - def _openai_accepts_reasoning_field( - key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None] + key: str, + value: object, + values_by_field: Mapping[str, frozenset[str | None] | None], + *, + keeps_unlisted_keys: bool, ) -> bool: if key == "effort": return True if key not in values_by_field: - return False + return keeps_unlisted_keys allowed: Final = values_by_field[key] return allowed is None or (isinstance(value, str | None) and value in allowed) def _reasoning_with_accepted_fields( - reasoning: Mapping[str, object], values_by_field: Mapping[str, frozenset[str | None] | None] + reasoning: Mapping[str, object], + values_by_field: Mapping[str, frozenset[str | None] | None], + *, + keeps_unlisted_keys: bool = False, ) -> Mapping[str, object]: return { - key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field) + key: value + for key, value in reasoning.items() + if _openai_accepts_reasoning_field(key, value, values_by_field, keeps_unlisted_keys=keeps_unlisted_keys) } def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model)) + model_info: Final = _openai_model_info(model) + if model_info is None: + return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=True) + if not model_info.get("supports_reasoning"): + return _reasoning_with_accepted_fields(reasoning, MappingProxyType({})) + return _reasoning_with_accepted_fields(reasoning, _openai_declared_reasoning_field_values(model_info)) -def openai_api_well_formed_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object]: - return _reasoning_with_accepted_fields(reasoning, _openai_declared_reasoning_field_values(None)) +def openai_api_well_formed_reasoning( + reasoning: Mapping[str, object], *, keeps_unlisted_keys: bool +) -> Mapping[str, object]: + return _reasoning_with_accepted_fields( + reasoning, _openai_declared_reasoning_field_values(None), keeps_unlisted_keys=keeps_unlisted_keys + ) def openai_well_formed_reasoning( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index d693df4d72c..031f27a6699 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,5 +1,6 @@ import json import logging +import time from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, cast @@ -12,6 +13,7 @@ from openai.types.responses.response_compaction_item import ResponseCompactionIt from pydantic import JsonValue, TypeAdapter import litellm +from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig @@ -36,7 +38,6 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import Choices, Message, ModelResponse from tests.unit.proxy.conftest import httpx_transport -import time _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' @@ -3348,6 +3349,9 @@ class TestMalformedParamsFollowDropParams: pytest.param({"parallel_tool_calls": "true"}, id="parallel-tool-calls-string"), pytest.param({"previous_response_id": 12}, id="previous-response-id-int"), pytest.param({"previous_response_id": ""}, id="previous-response-id-empty"), + pytest.param({"tool_choice": 5, "tools": [_WEATHER_TOOL]}, id="tool-choice-a-number-with-tools"), + pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="tool-choice-a-list-with-tools"), + pytest.param({"reasoning": "high"}, id="reasoning-not-an-object"), ], ) def test_a_malformed_value_is_dropped_under_drop_params_and_sent_as_is_without( @@ -3419,7 +3423,6 @@ class TestMalformedParamsFollowDropParams: "params", [ pytest.param({"tool_choice": {"name": "get_time"}, "tools": [_WEATHER_TOOL]}, id="choice-without-a-type"), - pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="choice-not-an-object"), pytest.param( {"tool_choice": {"type": "function", "name": "get_time"}, "tools": _WEATHER_TOOL}, id="tools-not-a-list" ), @@ -3536,11 +3539,10 @@ class TestMalformedParamsFollowDropParams: [ (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (AzureOpenAIOSeriesResponsesAPIConfig(), "o_series/gpt-6.1-sol"), - (OpenAIResponsesAPIConfig(), "my-fine-tuned-model"), ], - ids=["azure", "azure-o-series", "outside-openai-catalog"], + ids=["azure", "azure-o-series"], ) - def test_reasoning_values_no_openai_row_declares_are_dropped_where_the_openai_catalog_does_not_apply( + def test_azure_drops_reasoning_values_and_keys_no_openai_row_declares( self, local_model_cost_map, monkeypatch, config, model ): monkeypatch.setattr(litellm, "drop_params", False) @@ -3567,6 +3569,33 @@ class TestMalformedParamsFollowDropParams: assert kept["reasoning"] == declared assert sent["reasoning"] == malformed + def test_an_openai_model_outside_the_catalog_keeps_reasoning_keys_openai_does_not_define( + self, local_model_cost_map, monkeypatch + ): + monkeypatch.setattr(litellm, "drop_params", False) + server_keys: Final = {"max_tokens": 2000, "exclude": True} + requested: Final = {"effort": "high", "mode": "turbo", "context": "forever", "summary": "sometimes", **server_keys} + + dropped: Final = OpenAIResponsesAPIConfig().map_openai_params( + {"reasoning": dict(requested)}, "my-org/qwen3-32b", True + ) + + assert dropped["reasoning"] == {"effort": "high", **server_keys} + + def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map): + row: Final = litellm.model_cost["gpt-6.1-sol"] + new_mode: Final = "-".join(row["reasoning_modes"]) + "-next" + request: Final = {"reasoning": {"effort": "high", "mode": new_mode}} + + before_reload: Final = AzureOpenAIResponsesAPIConfig().map_openai_params(dict(request), "gpt-6.1-sol", True) + adopt_model_cost_map( + {**litellm.model_cost, "gpt-6.1-sol": {**row, "reasoning_modes": [*row["reasoning_modes"], new_mode]}} + ) + after_reload: Final = AzureOpenAIResponsesAPIConfig().map_openai_params(dict(request), "gpt-6.1-sol", True) + + assert before_reload["reasoning"] == {"effort": "high"} + assert after_reload == request + @pytest.mark.asyncio async def test_openai_responses_litellm_router_no_metadata(): diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index a9db0127ca6..b4c67d083c3 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1039,11 +1039,11 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr assert bridged(True) == {"effort": "high", "summary": "auto"} assert bridged(False) == requested -def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_outside_the_openai_catalog( +def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_keeps_other_keys( local_model_cost_map, ): declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] - requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "generate_summary": "auto"} + requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "max_tokens": 2000} def bridged(drop_params: bool) -> object: return openai_bridge_reasoning( @@ -1055,7 +1055,7 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_outside_ additional_drop_params=None, ) - assert bridged(True) == {"effort": "high", "mode": declared_mode} + assert bridged(True) == {"effort": "high", "mode": declared_mode, "max_tokens": 2000} assert bridged(False) == requested From 47e6144288bd0ebedd041896ed37ee51c9f77b8b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:27:04 -0700 Subject: [PATCH 6/9] fix(responses): drop a malformed tool_choice sent with a stored prompt under drop_params A stored prompt can carry its own tools, so the check kept any tool_choice next to one. A value that is not 'auto', 'none', 'required' or an object is malformed whatever tools the prompt holds, so it now drops under drop_params. Also types the parameters of the tests this PR adds. --- .../llms/openai/responses/transformation.py | 11 +++- .../count_tokens/test_token_counter.py | 4 +- ...test_openai_count_tokens_transformation.py | 4 +- .../test_openai_responses_transformation.py | 58 +++++++++++++------ tests/unit/responses/test_responses_utils.py | 14 ++--- 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index e6dd20ef15b..340f158da46 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -319,6 +319,7 @@ def _bundled_openai_reasoning_models() -> frozenset[str]: _TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"}) +_TOOL_CHOICE_STRINGS: Final = _TOOL_CHOICES_WITHOUT_TOOLS | {"required"} _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE) @@ -355,9 +356,17 @@ def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], . return True +def _has_tool_choice_shape(tool_choice: object) -> bool: + if isinstance(tool_choice, str): + return tool_choice in _TOOL_CHOICE_STRINGS + return as_str_mapping(tool_choice) is not None + + def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: - if tool_choice is None or params.get("prompt") is not None: + if tool_choice is None: return True + if params.get("prompt") is not None: + return _has_tool_choice_shape(tool_choice) tools: Final = params.get("tools") if isinstance(tool_choice, str): return tool_choice in _TOOL_CHOICES_WITHOUT_TOOLS or (tool_choice == "required" and bool(tools)) diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index cfb52bf9173..13c3c16e074 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -765,7 +765,7 @@ async def test_native_count_answers_openais_rejection_with_its_status_and_text( @pytest.mark.asyncio @pytest.mark.parametrize(("declared", "effort_counted"), [(True, True), (False, False)], ids=["declared", "undeclared"]) async def test_the_count_reads_the_deployments_declared_efforts_under_drop_params( - local_model_cost_map, monkeypatch: pytest.MonkeyPatch, declared: bool, effort_counted: bool + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, declared: bool, effort_counted: bool ) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] effort: Final = next( @@ -790,7 +790,7 @@ async def test_the_count_reads_the_deployments_declared_efforts_under_drop_param @pytest.mark.asyncio @pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "send-as-is"]) async def test_the_count_rounds_the_effort_to_a_declared_level_as_inference_does( - local_model_cost_map, monkeypatch: pytest.MonkeyPatch, drop_params: bool + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_params: bool ) -> None: declared: Final = ["low", "medium", "high"] monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 41e0c67348a..628ae7d1a83 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -683,7 +683,7 @@ def _disabled_effort(model: str) -> str: _DROP: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) -def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost_map) -> None: +def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost_map: None) -> None: malformed: Final = { "tool_choice": "required", "parallel_tool_calls": "true", @@ -700,7 +700,7 @@ def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost assert inference == {"reasoning": {"summary": "auto"}} -def test_the_count_keeps_an_effort_the_deployment_declares(local_model_cost_map) -> None: +def test_the_count_keeps_an_effort_the_deployment_declares(local_model_cost_map: None) -> None: effort: Final = _disabled_effort("gpt-6.1-sol") body: Final = native_count_request( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 031f27a6699..7fac30486df 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3352,11 +3352,21 @@ class TestMalformedParamsFollowDropParams: pytest.param({"tool_choice": 5, "tools": [_WEATHER_TOOL]}, id="tool-choice-a-number-with-tools"), pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="tool-choice-a-list-with-tools"), pytest.param({"reasoning": "high"}, id="reasoning-not-an-object"), + pytest.param( + {"tool_choice": 5, "prompt": {"id": "pmpt_abc"}}, id="tool-choice-a-number-with-stored-prompt" + ), + pytest.param( + {"tool_choice": "always", "prompt": {"id": "pmpt_abc"}}, id="unknown-string-with-stored-prompt" + ), ], ) def test_a_malformed_value_is_dropped_under_drop_params_and_sent_as_is_without( - self, local_model_cost_map, monkeypatch, config, params - ): + self, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + config: OpenAIResponsesAPIConfig, + params: dict[str, object], + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) malformed: Final = next(name for name in params if name != "tools") @@ -3367,8 +3377,8 @@ class TestMalformedParamsFollowDropParams: assert sent == params def test_azure_o_series_forwards_a_row_disabled_effort_while_dropping_malformed_values( - self, local_model_cost_map, monkeypatch - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**row, "supports_minimal_reasoning_effort": False}) @@ -3392,6 +3402,10 @@ class TestMalformedParamsFollowDropParams: pytest.param({"parallel_tool_calls": None}, id="parallel-tool-calls-null"), pytest.param({"previous_response_id": "resp_abc"}, id="previous-response-id-string"), pytest.param({"tool_choice": "required", "prompt": {"id": "pmpt_abc"}}, id="required-with-stored-prompt"), + pytest.param( + {"tool_choice": {"type": "function", "name": "lookup_order"}, "prompt": {"id": "pmpt_abc"}}, + id="function-from-stored-prompt", + ), pytest.param( {"tool_choice": {"type": "custom", "name": "other"}, "tools": [_SQL_TOOL]}, id="custom-with-a-custom-tool", @@ -3416,7 +3430,9 @@ class TestMalformedParamsFollowDropParams: ), ], ) - def test_a_well_formed_value_is_kept_under_drop_params(self, local_model_cost_map, params): + def test_a_well_formed_value_is_kept_under_drop_params( + self, local_model_cost_map: None, params: dict[str, object] + ) -> None: assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params @pytest.mark.parametrize( @@ -3428,10 +3444,12 @@ class TestMalformedParamsFollowDropParams: ), ], ) - def test_a_tool_choice_the_check_cannot_read_is_kept_under_drop_params(self, local_model_cost_map, params): + def test_a_tool_choice_the_check_cannot_read_is_kept_under_drop_params( + self, local_model_cost_map: None, params: dict[str, object] + ) -> None: assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params - def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map): + def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map: None) -> None: params: Final = { "tool_choice": "required", "parallel_tool_calls": "true", @@ -3446,8 +3464,8 @@ class TestMalformedParamsFollowDropParams: [("mode", "turbo"), ("context", "forever"), ("summary", "sometimes"), ("generate_summary", "auto")], ) def test_a_reasoning_value_the_model_row_does_not_declare_is_dropped_under_drop_params( - self, local_model_cost_map, monkeypatch, field, value - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, field: str, value: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) row: Final = litellm.model_cost["gpt-6.1-sol"] declared: Final = { @@ -3466,8 +3484,8 @@ class TestMalformedParamsFollowDropParams: @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) def test_a_row_without_reasoning_declarations_drops_values_no_openai_row_declares( - self, local_model_cost_map, monkeypatch, field, declarations - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, field: str, declarations: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) undeclared_model: Final = next( name @@ -3487,7 +3505,9 @@ class TestMalformedParamsFollowDropParams: assert sent["reasoning"] == malformed assert kept["reasoning"] == declared - def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params(self, local_model_cost_map, monkeypatch): + def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) non_reasoning_model: Final = next( name @@ -3515,8 +3535,8 @@ class TestMalformedParamsFollowDropParams: ids=["openai", "azure", "azure-o-series"], ) def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it( - self, local_model_cost_map, monkeypatch, config, model - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, config: OpenAIResponsesAPIConfig, model: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) reasoning: Final = {"effort": "not-an-effort", "summary": "auto"} @@ -3543,8 +3563,8 @@ class TestMalformedParamsFollowDropParams: ids=["azure", "azure-o-series"], ) def test_azure_drops_reasoning_values_and_keys_no_openai_row_declares( - self, local_model_cost_map, monkeypatch, config, model - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, config: OpenAIResponsesAPIConfig, model: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) row: Final = litellm.model_cost["gpt-6.1-sol"] declared: Final = { @@ -3570,8 +3590,8 @@ class TestMalformedParamsFollowDropParams: assert sent["reasoning"] == malformed def test_an_openai_model_outside_the_catalog_keeps_reasoning_keys_openai_does_not_define( - self, local_model_cost_map, monkeypatch - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) server_keys: Final = {"max_tokens": 2000, "exclude": True} requested: Final = {"effort": "high", "mode": "turbo", "context": "forever", "summary": "sometimes", **server_keys} @@ -3582,7 +3602,7 @@ class TestMalformedParamsFollowDropParams: assert dropped["reasoning"] == {"effort": "high", **server_keys} - def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map): + def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map: None) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] new_mode: Final = "-".join(row["reasoning_modes"]) + "-next" request: Final = {"reasoning": {"effort": "high", "mode": new_mode}} diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index d32fcfd47ea..0b062c4c75b 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1014,8 +1014,8 @@ def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_drop_params( - local_model_cost_map, field, declarations -): + local_model_cost_map: None, field: str, declarations: str +) -> None: undeclared_model: Final = next( name for name, row in litellm.model_cost.items() @@ -1044,8 +1044,8 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr assert bridged(False) == requested def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_keeps_other_keys( - local_model_cost_map, -): + local_model_cost_map: None, +) -> None: declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "max_tokens": 2000} @@ -1068,8 +1068,8 @@ def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_ def test_the_chat_bridge_keeps_other_keys_for_an_openai_prefixed_name_it_resolves_to_a_catalog_row( - local_model_cost_map, -): + local_model_cost_map: None, +) -> None: declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] requested: Final = {"effort": "high", "mode": "not-a-declared-value", "max_tokens": 2000} @@ -1092,7 +1092,7 @@ def test_the_chat_bridge_keeps_other_keys_for_an_openai_prefixed_name_it_resolve assert bridged("gpt-6.1-sol", True) == {"effort": "high", "mode": declared_mode} -def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map): +def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map: None) -> None: non_reasoning_model: Final = next( name for name, row in litellm.model_cost.items() From f9c805e77f9d9b3c5f6cb2ba0942b7e44fa06569 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:41:55 -0700 Subject: [PATCH 7/9] test(responses): cover a null tool_choice and a provider that declares no reasoning values Adds behavior tests for the two added lines diff coverage found unrun. A null tool_choice stays under drop_params, and the chat bridge keeps every reasoning key for a provider that declares no values. --- .../test_openai_responses_transformation.py | 1 + tests/unit/responses/test_responses_utils.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 7fac30486df..bed173388c2 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3393,6 +3393,7 @@ class TestMalformedParamsFollowDropParams: [ pytest.param({"tool_choice": "auto"}, id="auto-without-tools"), pytest.param({"tool_choice": "none"}, id="none-without-tools"), + pytest.param({"tool_choice": None}, id="tool-choice-null"), pytest.param({"tool_choice": "required", "tools": [_WEATHER_TOOL]}, id="required-with-tools"), pytest.param( {"tool_choice": {"type": "function", "name": "get_weather"}, "tools": [_WEATHER_TOOL]}, diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 0b062c4c75b..63bab585612 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1092,6 +1092,27 @@ def test_the_chat_bridge_keeps_other_keys_for_an_openai_prefixed_name_it_resolve assert bridged("gpt-6.1-sol", True) == {"effort": "high", "mode": declared_mode} +def test_the_chat_bridge_keeps_every_reasoning_value_for_a_provider_that_declares_none( + local_model_cost_map: None, +) -> None: + requested: Final = {"effort": "high", "mode": "turbo", "context": "forever"} + + bridged: Final = bridge_reasoning( + requested=requested, + effort_kept=True, + summary_alias=None, + model="my-vllm-model", + base_model=None, + custom_llm_provider="hosted_vllm", + api_base=None, + deployment_model_info=None, + drop_params=True, + additional_drop_params=None, + ) + + assert bridged == requested + + def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map: None) -> None: non_reasoning_model: Final = next( name From eb1d8919b9ee7fa7dbfad67ae7321b4e73bd515d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:06:46 -0700 Subject: [PATCH 8/9] fix(responses): check allowed_tools entries without recursion and drop a nested allowed_tools The repo's recursion check refuses recursive helpers, so allowed_tools entries now go through a separate per-entry check. OpenAI rejects an allowed_tools entry nested inside allowed_tools with a 400, so that choice now drops under drop_params. --- litellm/llms/openai/responses/transformation.py | 14 +++++++++++--- .../test_openai_responses_transformation.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 340f158da46..193bfc25b33 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -337,11 +337,19 @@ def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], . choice: Final = as_str_mapping(tool_choice) if choice is None: return False + if choice.get("type") != "allowed_tools": + return _sent_tools_include(choice, tools) + allowed: Final = choice.get("tools") + entries: Final = tuple(map(as_str_mapping, _json_array_or_empty(allowed))) + return not isinstance(allowed, list | tuple) or all( + entry is not None and _sent_tools_include(entry, tools) for entry in entries + ) + + +def _sent_tools_include(choice: Mapping[str, object], tools: tuple[Mapping[str, object], ...]) -> bool: match choice.get("type"): case "allowed_tools": - allowed: Final = choice.get("tools") - entries: Final = _json_array_or_empty(allowed) - return not isinstance(allowed, list | tuple) or all(_names_a_sent_tool(entry, tools) for entry in entries) + return False case "function": return any(tool.get("type") == "function" and tool.get("name") == choice.get("name") for tool in tools) case "custom": diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index bed173388c2..38d848086b5 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3346,6 +3346,23 @@ class TestMalformedParamsFollowDropParams: }, id="allowed-tools-naming-a-tool-not-sent", ), + pytest.param( + { + "tool_choice": { + "type": "allowed_tools", + "mode": "auto", + "tools": [ + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}], + } + ], + }, + "tools": [_WEATHER_TOOL, {"type": "web_search"}], + }, + id="allowed-tools-nested-in-allowed-tools", + ), pytest.param({"parallel_tool_calls": "true"}, id="parallel-tool-calls-string"), pytest.param({"previous_response_id": 12}, id="previous-response-id-int"), pytest.param({"previous_response_id": ""}, id="previous-response-id-empty"), From 0a0b322849955fd3c5dbb8d3f84a9baf2d600d0c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:31:21 -0700 Subject: [PATCH 9/9] fix(responses): import the reasoning effort capability from its litellm_core_utils path --- .../llms/openai/responses/count_tokens/token_counter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 71f55dcecfd..9849b06cc6e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -14,6 +14,10 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.reasoning_effort_capability import ( + deployment_reasoning_effort_levels, + resolve_deployment_reasoning_effort, +) from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error @@ -37,10 +41,6 @@ from litellm.llms.openai.workload_identity import ( openai_static_api_key, resolve_openai_workload_identity_config, ) -from litellm.router_utils.reasoning_effort_capability import ( - deployment_reasoning_effort_levels, - resolve_deployment_reasoning_effort, -) from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests