diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 9cba5db8ab7..ba8738c8de0 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -53,12 +53,14 @@ class GetModelCostMap: _backup_model_count: int = -1 # -1 = not yet loaded + @staticmethod + def read_local_model_cost_map_text() -> str: + return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") + @staticmethod def load_local_model_cost_map() -> dict: """Load the local backup model cost map bundled with the package.""" - content: Final = json.loads( - files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") - ) + content: Final = json.loads(GetModelCostMap.read_local_model_cost_map_text()) return content @classmethod diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 123e9a1dda4..189cc7fe956 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,15 +1,17 @@ from collections.abc import Mapping, Sequence +from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints import httpx from openai.types.responses import ResponseReasoningItem -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -42,6 +44,30 @@ _PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR: Final = frozenset({LlmProviders. _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) +class _ReasoningSupportEntry(BaseModel): + litellm_provider: str | None = None + supports_reasoning: bool | None = None + + +_BUNDLED_COST_MAP: Final = TypeAdapter(dict[str, _ReasoningSupportEntry]) + + +@lru_cache(maxsize=1) +def _bundled_openai_reasoning_models() -> frozenset[str]: + """OpenAI models the cost map shipped with this release flags as reasoning models. + + The live map can lag this release (a pinned mirror, or a proxy on newer code than the + map it fetches), and a lagging entry must never strip `reasoning` from a model this + release knows accepts it. + """ + bundled: Final = _BUNDLED_COST_MAP.validate_json(GetModelCostMap.read_local_model_cost_map_text()) + return frozenset( + name + for name, entry in bundled.items() + if entry.litellm_provider == LlmProviders.OPENAI.value and entry.supports_reasoning is True + ) + + class _DeleteResponseBody(TypedDict): """Decoded body of the Responses API delete call.""" @@ -95,13 +121,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _supports_reasoning_effort_none(model: str) -> bool: """Return True if the model supports reasoning.effort='none'.""" - from litellm.utils import _supports_factory + from litellm.utils import supports_none_reasoning_effort - return _supports_factory( - model=model, - custom_llm_provider=None, - key="supports_none_reasoning_effort", - ) + return supports_none_reasoning_effort(model=model, custom_llm_provider=None) @staticmethod def _effort_resolves_to_none(model: str, effort: str | None) -> bool: @@ -117,11 +139,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return OpenAIGPT5Config.effort_resolves_to_none(model, effort) - def _supports_reasoning_param(self, model: str) -> bool: - base: Final = model.split("/")[-1] - if base not in litellm.open_ai_chat_completion_models: + @staticmethod + def _supports_reasoning_param(model: str) -> bool: + from litellm.utils import _get_model_info_helper + + try: + info: Final = _get_model_info_helper( + model=model.split("/")[-1], custom_llm_provider=LlmProviders.OPENAI.value + ) + except Exception: return True - return litellm.supports_reasoning(model=base, custom_llm_provider=self.custom_llm_provider.value) + return info["key"] in _bundled_openai_reasoning_models() or info.get("supports_reasoning") is True @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -182,7 +210,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): else: raise litellm.UnsupportedParamsError( message=( - f"{model} doesn't support the `reasoning` parameter. " + f"{model} doesn't support the `reasoning` parameter " + "(its model cost map entry lacks `supports_reasoning`). " "To drop unsupported params set `litellm.drop_params = True`" ), status_code=400, @@ -500,7 +529,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): processed_headers: Final = process_response_headers(raw_response_headers) try: response = ResponsesAPIResponse.model_validate(raw_response_json) - except Exception: + except ValidationError: verbose_logger.debug( "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json ) @@ -892,7 +921,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) - except Exception: + except ValidationError: verbose_logger.debug( "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 99e728a30bc..c97858b4e29 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30698,6 +30698,7 @@ "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/litellm/utils.py b/litellm/utils.py index 9d20d32d147..07735cc1f87 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2805,6 +2805,13 @@ def supports_reasoning(model: str, custom_llm_provider: str | None = None) -> bo return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_reasoning") +def supports_none_reasoning_effort(model: str, custom_llm_provider: str | None = None) -> bool: + """ + Check if the given model accepts reasoning effort "none" and return a boolean value. + """ + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_none_reasoning_effort") + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -5210,13 +5217,16 @@ def _strip_openai_finetune_model_name(model_name: str) -> str: input: ft:gpt-3.5-turbo:my-org:custom_suffix:id output: ft:gpt-3.5-turbo + input: ft:gpt-4o-2024-08-06:my-org::id (OpenAI leaves the suffix empty when none was set) + output: ft:gpt-4o-2024-08-06 + Args: model_name (str): The full model name Returns: str: The stripped model name """ - return re.sub(r"(:[^:]+){3}$", "", model_name) + return re.sub(r"(:[^:]*){3}$", "", model_name) def _strip_model_name(model: str, custom_llm_provider: str | None) -> str: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 99e728a30bc..c97858b4e29 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30698,6 +30698,7 @@ "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index cc884fd7dc1..66271de7d04 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -2034,12 +2034,14 @@ class TestReasoningFollowsModelSupport: ("gpt-4o", False), ("gpt-4.1", False), ("gpt-4o-mini", False), - ("gpt-5-search-api", False), + ("ft:gpt-4o-2024-08-06:my-org::abc123", False), + ("chat-latest", True), ("gpt-5.6", True), ("o3", True), ("o3-deep-research", True), ("o4-mini-deep-research", True), ("codex-mini-latest", True), + ("ft:o4-mini-2025-04-16:my-org::abc123", True), ("computer-use-preview", True), ], ) @@ -2051,6 +2053,30 @@ class TestReasoningFollowsModelSupport: ) assert ("reasoning" in mapped) is reasoning_survives + @pytest.mark.parametrize( + "model, reasoning_survives", + [ + ("gpt-4o", False), + ("chat-latest", True), + ("o3", True), + ("o3-deep-research", True), + ], + ) + def test_a_cost_map_older_than_this_release_never_strips_a_known_reasoning_model( + self, local_model_cost_map, monkeypatch, model, reasoning_survives + ): + lagging = { + name: {field: value for field, value in entry.items() if field != "supports_reasoning"} + for name, entry in litellm.model_cost.items() + } + monkeypatch.setattr(litellm, "model_cost", lagging) + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model=model, + drop_params=True, + ) + assert ("reasoning" in mapped) is reasoning_survives + def test_without_drop_params_the_error_is_litellms_400(self, local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) with pytest.raises(litellm.UnsupportedParamsError) as excinfo: @@ -2060,6 +2086,7 @@ class TestReasoningFollowsModelSupport: drop_params=False, ) assert excinfo.value.status_code == 400 + assert "cost map" in str(excinfo.value) def test_azure_deployments_keep_reasoning(self, local_model_cost_map): mapped = AzureOpenAIResponsesAPIConfig().map_openai_params( diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 79609032fbd..3f0992275eb 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -8,6 +8,8 @@ from pathlib import Path import jsonschema import pytest +from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name + REPO_ROOT = Path(__file__).parents[2] GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -175,22 +177,35 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict): ) +OPENAI_REASONING_FAMILY_MARKERS = ("codex", "deep-research", "chat-latest") + + def is_openai_o_series(name: str) -> bool: - base = name.split("/")[-1] - return len(base) > 1 and base[0] == "o" and base[1].isdigit() + return len(name) > 1 and name[0] == "o" and name[1].isdigit() -def test_openai_o_series_entries_carry_supports_reasoning(prices: dict): +def is_openai_reasoning_family(name: str) -> bool: + base = name.split("/")[-1].removeprefix("ft:") + if "search-api" in base: + return False + return ( + is_openai_o_series(base) + or is_gpt_reasoning_series_name(base) + or any(marker in base for marker in OPENAI_REASONING_FAMILY_MARKERS) + ) + + +def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict): unflagged = [ name for name, entry in prices.items() if isinstance(entry, dict) and entry.get("litellm_provider") == "openai" - and is_openai_o_series(name) + and is_openai_reasoning_family(name) and entry.get("supports_reasoning") is not True ] assert unflagged == [], ( - "OpenAI o-series models are reasoning models, and the Responses API drops the " - "`reasoning` param for any mapped OpenAI model whose entry lacks supports_reasoning; " - "flag these entries:\n" + "\n".join(unflagged) + "OpenAI o-series, gpt-5+, codex, deep-research, and chat-latest models are reasoning " + "models, and the Responses API drops the `reasoning` param for any mapped OpenAI model " + "whose entry lacks supports_reasoning; flag these entries:\n" + "\n".join(unflagged) ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 14907e17b1b..6a90593bea9 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -198,6 +198,11 @@ def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_ma assert via_provider["mode"] == "responses" +def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map): + info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai") + assert info["key"] == "ft:gpt-4o-2024-08-06" + + def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map): """The provider-prefixed candidate is tried last, after every candidate that already existed, so no model that resolves today can change answer. `perplexity/sonar`