diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 259a94bd14d..59abafea3f8 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -65,9 +65,7 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): """ if not drop_params: return dict(response_api_optional_params) - kept: Final = self.params_dropped_for_deployment( - response_api_optional_params, model, declared_reasoning_efforts=() - ) + kept: Final = self._well_formed_params(response_api_optional_params) if "temperature" in kept: verbose_logger.debug( "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 88ce5002803..bef80dd9612 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -35,7 +35,10 @@ from litellm.llms.openai.workload_identity import ( openai_static_api_key, resolve_openai_workload_identity_config, ) -from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels +from litellm.router_utils.reasoning_effort_capability import ( + deployment_reasoning_effort_levels, + resolve_deployment_reasoning_effort, +) from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests @@ -96,6 +99,24 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +def _with_deployment_reasoning( + request: Mapping[str, object], model: str, model_info: object, policy: DropPolicy +) -> Mapping[str, object]: + reasoning: Final = as_str_mapping(request.get("reasoning")) + if reasoning is None: + return request + resolved: Final = resolve_deployment_reasoning_effort( + reasoning, + model=model, + custom_llm_provider=LlmProviders.OPENAI.value, + deployment_model_info=model_info, + model_group=None, + drop_params=policy.drop_params, + ).value + without_reasoning: Final = {field: value for field, value in request.items() if field != "reasoning"} + return without_reasoning if resolved is None else {**without_reasoning, "reasoning": resolved} + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -219,14 +240,12 @@ class OpenAITokenCounter(BaseTokenCounter): resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) + model_info: Final = None if deployment is None else deployment.get("model_info") body: Final = native_count_request( - request, + _with_deployment_reasoning(request, model_to_use, model_info, policy), model_to_use, policy, - declared_reasoning_efforts=deployment_reasoning_effort_levels( - None if deployment is None else deployment.get("model_info") - ) - or (), + declared_reasoning_efforts=deployment_reasoning_effort_levels(model_info) or (), ) return await self._counted( resolved, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index c628d02a070..2a77291e6dc 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -2,7 +2,7 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_type_hints +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, get_args, get_type_hints import httpx from openai.types.responses import ResponseReasoningItem @@ -50,6 +50,7 @@ from litellm.responses.utils import openai_well_formed_reasoning from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * +from litellm.types.llms.openai import REASONING_EFFORT from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -321,6 +322,7 @@ _TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"}) _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE) +_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: @@ -499,19 +501,27 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return None def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - if self.custom_llm_provider == LlmProviders.OPENAI and OpenAIGPTConfig.is_openai_catalog_model(model): - return openai_well_formed_reasoning(reasoning, model) - return reasoning + if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model): + return reasoning + return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model)) + + def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool: + if effort is None or self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: + return True + return isinstance(effort, str) and ( + effort in _REASONING_EFFORTS or effort in (declared_reasoning_efforts or ()) + ) + + def _well_formed_params(self, params: Mapping[str, object]) -> Mapping[str, object]: + if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: + return params + return {name: value for name, value in params.items() if _is_well_formed(name, value, params)} def params_dropped_for_deployment( self, params: Mapping[str, object], model: str, *, declared_reasoning_efforts: Collection[str] | None ) -> Mapping[str, object]: """The request params this deployment forwards under drop_params, malformed values removed.""" - well_formed: Final = ( - {name: value for name, value in params.items() if _is_well_formed(name, value, params)} - if self.custom_llm_provider in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES - else params - ) + well_formed: Final = self._well_formed_params(params) without_reasoning: Final = {name: value for name, value in well_formed.items() if name != "reasoning"} reasoning: Final = well_formed.get("reasoning") refusal: Final = self._reasoning_refusal(reasoning, model, declared_reasoning_efforts) @@ -520,7 +530,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): reasoning_fields: Final = as_str_mapping(reasoning) if reasoning_fields is None: return well_formed - effort_kept: Final = not isinstance(refusal, _EffortDisabled) + effort_kept: Final = not isinstance(refusal, _EffortDisabled) and self._is_well_formed_effort( + reasoning_fields.get("effort"), declared_reasoning_efforts + ) kept_reasoning: Final = self._accepted_reasoning( {key: value for key, value in reasoning_fields.items() if effort_kept or key != "effort"}, model ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 2cb6fc8b436..73933901000 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -2,11 +2,12 @@ import base64 import os import re from collections.abc import Callable, Collection, Iterable, Mapping, Sequence -from functools import reduce +from functools import lru_cache, reduce +from itertools import chain from types import MappingProxyType from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion @@ -14,6 +15,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -1412,6 +1414,7 @@ def reasoning_with_summary_default( _SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) +_NON_REASONING_MODEL_FIELD_VALUES: Final = MappingProxyType({"summary": _SUMMARY_VALUES}) def _openai_model_info(model: str) -> ModelInfo | None: @@ -1421,18 +1424,42 @@ def _openai_model_info(model: str) -> ModelInfo | None: return None if isinstance(model_info, ModelInfoLookupMiss) else model_info -def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | None: - return None if values is None else frozenset(values) +class _ReasoningDeclarations(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_provider: str | None = None + reasoning_modes: tuple[str, ...] | None = None + reasoning_contexts: tuple[str, ...] | None = None + + +_BUNDLED_REASONING_DECLARATIONS: Final = TypeAdapter(dict[str, _ReasoningDeclarations]) + + +@lru_cache(maxsize=1) +def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]: + bundled: Final = _BUNDLED_REASONING_DECLARATIONS.validate_json(GetModelCostMap.read_local_model_cost_map_text()) + openai_rows: Final = tuple(row for row in bundled.values() if row.litellm_provider == "openai") + return MappingProxyType( + { + "mode": frozenset(chain.from_iterable(row.reasoning_modes or () for row in openai_rows)), + "context": frozenset(chain.from_iterable(row.reasoning_contexts or () for row in openai_rows)), + } + ) + + +def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) -> frozenset[str | None]: + return undeclared if values is None else frozenset(values) def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Mapping[str, frozenset[str | None] | None]: if model_info is None: return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) + declared_anywhere: Final = _reasoning_values_any_openai_row_declares() return MappingProxyType( { "summary": _SUMMARY_VALUES, - "mode": _declared_values(model_info.get("reasoning_modes")), - "context": _declared_values(model_info.get("reasoning_contexts")), + "mode": _declared_values(model_info.get("reasoning_modes"), declared_anywhere["mode"]), + "context": _declared_values(model_info.get("reasoning_contexts"), declared_anywhere["context"]), } ) @@ -1467,10 +1494,15 @@ def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> M return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model)) -def openai_well_formed_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - return _reasoning_with_accepted_fields( - reasoning, _openai_declared_reasoning_field_values(_openai_model_info(model)) +def openai_well_formed_reasoning( + reasoning: Mapping[str, object], model: str, *, model_reasons: bool +) -> Mapping[str, object]: + values_by_field: Final = ( + _openai_declared_reasoning_field_values(_openai_model_info(model)) + if model_reasons + else _NON_REASONING_MODEL_FIELD_VALUES ) + return _reasoning_with_accepted_fields(reasoning, values_by_field) def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 6b298ed319b..7fb7f0cbded 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -667,3 +667,24 @@ async def test_the_count_reads_the_deployments_declared_efforts_under_drop_param sent: Final = json.loads(route.calls.last.request.content) assert ("reasoning" in sent) is effort_counted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "send-as-is"]) +async def test_the_count_rounds_the_effort_to_a_declared_level_as_inference_does( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, drop_params: bool +) -> None: + declared: Final = ["low", "medium", "high"] + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + await OpenAITokenCounter().count_responses_request( + model_to_use="gpt-6.1-sol", + request={"model": "my-gpt", "input": "hi", "reasoning": {"effort": "max", "summary": "auto"}}, + policy=DropPolicy(drop_params=drop_params, additional_drop_params=frozenset()), + deployment={"litellm_params": {"api_key": "sk-test"}, "model_info": {"reasoning_effort_levels": declared}}, + request_model="my-gpt", + ) + sent: Final = json.loads(route.calls.last.request.content) + + assert sent["reasoning"] == {"effort": "high", "summary": "auto"} diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 70213534caa..a4a930c42a2 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -18,6 +18,7 @@ from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIC from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig @@ -3336,6 +3337,18 @@ class TestMalformedParamsFollowDropParams: assert malformed not in dropped assert sent == params + def test_azure_o_series_forwards_a_row_disabled_effort_while_dropping_malformed_values( + self, local_model_cost_map, monkeypatch + ): + row: Final = litellm.model_cost["gpt-6.1-sol"] + monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**row, "supports_minimal_reasoning_effort": False}) + + mapped: Final = AzureOpenAIOSeriesResponsesAPIConfig().map_openai_params( + {"reasoning": {"effort": "minimal"}, "tool_choice": "required"}, "gpt-6.1-sol", True + ) + + assert mapped == {"reasoning": {"effort": "minimal"}} + @pytest.mark.parametrize( "params", [ @@ -3383,6 +3396,68 @@ class TestMalformedParamsFollowDropParams: assert dropped["reasoning"] == {key: kept for key, kept in declared.items() if key != field} assert sent["reasoning"] == reasoning + @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) + def test_a_row_without_reasoning_declarations_drops_values_no_openai_row_declares( + self, local_model_cost_map, monkeypatch, field, declarations + ): + monkeypatch.setattr(litellm, "drop_params", False) + undeclared_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" and row.get("supports_reasoning") and declarations not in row + ) + declared_elsewhere: Final = litellm.model_cost["gpt-6.1-sol"][declarations][0] + malformed: Final = {"summary": "auto", field: "not-a-declared-value"} + declared: Final = {"summary": "auto", field: declared_elsewhere} + config: Final = OpenAIResponsesAPIConfig() + + dropped: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, True) + sent: Final = config.map_openai_params({"reasoning": dict(malformed)}, undeclared_model, False) + kept: Final = config.map_openai_params({"reasoning": dict(declared)}, undeclared_model, True) + + assert dropped["reasoning"] == {"summary": "auto"} + assert sent["reasoning"] == malformed + assert kept["reasoning"] == declared + + def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params(self, local_model_cost_map, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + non_reasoning_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" + and not row.get("supports_reasoning") + and OpenAIGPTConfig.is_openai_catalog_model(name) + ) + request: Final = {"reasoning": {"summary": "auto", "mode": "standard", "context": "auto"}, "tool_choice": "auto"} + config: Final = OpenAIResponsesAPIConfig() + + dropped: Final = config.map_openai_params(dict(request), non_reasoning_model, True) + sent: Final = config.map_openai_params(dict(request), non_reasoning_model, False) + + assert dropped == {"reasoning": {"summary": "auto"}, "tool_choice": "auto"} + assert sent == request + + @pytest.mark.parametrize( + ("config", "model"), + [(OpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol")], + ids=["openai", "azure"], + ) + def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it( + self, local_model_cost_map, monkeypatch, config, model + ): + monkeypatch.setattr(litellm, "drop_params", False) + reasoning: Final = {"effort": "not-an-effort", "summary": "auto"} + + dropped: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, True) + sent: Final = config.map_openai_params({"reasoning": dict(reasoning)}, model, False) + declared: Final = config.map_openai_params_for_deployment( + {"reasoning": dict(reasoning)}, model, True, declared_reasoning_efforts=("not-an-effort",) + ) + + assert dropped["reasoning"] == {"summary": "auto"} + assert sent["reasoning"] == reasoning + assert declared["reasoning"] == reasoning + @pytest.mark.parametrize( ("config", "model"), [(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), (OpenAIResponsesAPIConfig(), "my-fine-tuned-model")], diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 0863abd1f1e..5e6c98267fb 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -10,7 +10,8 @@ import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig -from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils, openai_bridge_reasoning from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.utils import ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map @@ -1009,3 +1010,31 @@ def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): else: litellm.model_cost[model] = previous_row _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) +def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_drop_params( + local_model_cost_map, field, declarations +): + undeclared_model: Final = next( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "openai" + and row.get("supports_reasoning") + and declarations not in row + and OpenAIGPTConfig.is_openai_catalog_model(name) + ) + requested: Final = {"effort": "high", "summary": "auto", field: "not-a-declared-value"} + + def bridged(drop_params: bool) -> object: + return openai_bridge_reasoning( + requested=requested, + effort_kept=True, + summary_alias=None, + model=undeclared_model, + drop_params=drop_params, + additional_drop_params=None, + ) + + assert bridged(True) == {"effort": "high", "summary": "auto"} + assert bridged(False) == requested