mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses): keep off-catalog reasoning keys, read reasoning lists from the loaded cost map, drop non-object reasoning and tool_choice
This commit is contained in:
parent
e616590e58
commit
0967601865
4 changed files with 99 additions and 39 deletions
|
|
@ -335,7 +335,7 @@ def _tool_mappings(tools: object) -> tuple[Mapping[str, object], ...]:
|
|||
def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], ...]) -> bool:
|
||||
choice: Final = as_str_mapping(tool_choice)
|
||||
if choice is None:
|
||||
return True
|
||||
return False
|
||||
match choice.get("type"):
|
||||
case "allowed_tools":
|
||||
allowed: Final = choice.get("tools")
|
||||
|
|
@ -375,6 +375,8 @@ def _is_well_formed(name: str, value: object, params: Mapping[str, object]) -> b
|
|||
return value is None or isinstance(value, bool)
|
||||
case "previous_response_id":
|
||||
return value is None or (isinstance(value, str) and value != "")
|
||||
case "reasoning":
|
||||
return value is None or as_str_mapping(value) is not None
|
||||
case _:
|
||||
return True
|
||||
|
||||
|
|
@ -536,8 +538,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
def _accepted_reasoning(self, reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
|
||||
if self.custom_llm_provider not in _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES:
|
||||
return reasoning
|
||||
if self.custom_llm_provider != LlmProviders.OPENAI or not OpenAIGPTConfig.is_openai_catalog_model(model):
|
||||
return openai_api_well_formed_reasoning(reasoning)
|
||||
if self.custom_llm_provider != LlmProviders.OPENAI:
|
||||
return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=False)
|
||||
if not OpenAIGPTConfig.is_openai_catalog_model(model):
|
||||
return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=True)
|
||||
return openai_well_formed_reasoning(reasoning, model, model_reasons=self._supports_reasoning_param(model=model))
|
||||
|
||||
def _is_well_formed_effort(self, effort: object, declared_reasoning_efforts: Collection[str] | None) -> bool:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from itertools import chain
|
|||
from types import MappingProxyType
|
||||
from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import TypedDict as ExtraItemsTypedDict
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion
|
||||
|
||||
|
|
@ -15,7 +15,6 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -1423,29 +1422,46 @@ def _openai_model_info(model: str) -> ModelInfo | None:
|
|||
return None if isinstance(model_info, ModelInfoLookupMiss) else model_info
|
||||
|
||||
|
||||
class _ReasoningDeclarations(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
litellm_provider: str | None = None
|
||||
reasoning_modes: tuple[str, ...] | None = None
|
||||
reasoning_contexts: tuple[str, ...] | None = None
|
||||
def _declared_strings(rows: Iterable[Mapping[str, object]], field: str) -> frozenset[str]:
|
||||
return frozenset(chain.from_iterable(_string_items(row.get(field)) for row in rows))
|
||||
|
||||
|
||||
_BUNDLED_REASONING_DECLARATIONS: Final = TypeAdapter(dict[str, _ReasoningDeclarations])
|
||||
def _string_items(values: object) -> tuple[str, ...]:
|
||||
listed: Final = iter_items(values) if type(values) in (list, tuple) else ()
|
||||
return tuple(value for value in listed if isinstance(value, str))
|
||||
|
||||
|
||||
_NO_COST_MAP: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _model_cost() -> Mapping[str, object]:
|
||||
model_cost: Final = as_str_mapping(attribute_or_none(litellm, "model_cost"))
|
||||
return _NO_COST_MAP if model_cost is None else model_cost
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]:
|
||||
bundled: Final = _BUNDLED_REASONING_DECLARATIONS.validate_json(GetModelCostMap.read_local_model_cost_map_text())
|
||||
openai_rows: Final = tuple(row for row in bundled.values() if row.litellm_provider == "openai")
|
||||
def _reasoning_values_declared_in_cost_map(
|
||||
_cost_map_id: int, _cost_map_generation: int
|
||||
) -> Mapping[str, frozenset[str]]:
|
||||
openai_rows: Final = tuple(
|
||||
row
|
||||
for row in map(as_str_mapping, tuple(_model_cost().values()))
|
||||
if row is not None and row.get("litellm_provider") == "openai"
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
"mode": frozenset(chain.from_iterable(row.reasoning_modes or () for row in openai_rows)),
|
||||
"context": frozenset(chain.from_iterable(row.reasoning_contexts or () for row in openai_rows)),
|
||||
"mode": _declared_strings(openai_rows, "reasoning_modes"),
|
||||
"context": _declared_strings(openai_rows, "reasoning_contexts"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _reasoning_values_any_openai_row_declares() -> Mapping[str, frozenset[str]]:
|
||||
from litellm.utils import get_model_cost_mutation_generation
|
||||
|
||||
return _reasoning_values_declared_in_cost_map(id(_model_cost()), get_model_cost_mutation_generation())
|
||||
|
||||
|
||||
def _declared_values(values: Sequence[str] | None, undeclared: frozenset[str]) -> frozenset[str | None]:
|
||||
return undeclared if values is None else frozenset(values)
|
||||
|
||||
|
|
@ -1463,38 +1479,49 @@ def _openai_declared_reasoning_field_values(model_info: ModelInfo | None) -> Map
|
|||
)
|
||||
|
||||
|
||||
def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]:
|
||||
model_info: Final = _openai_model_info(model)
|
||||
if model_info is not None and not model_info.get("supports_reasoning"):
|
||||
return MappingProxyType({})
|
||||
return _openai_declared_reasoning_field_values(model_info)
|
||||
|
||||
|
||||
def _openai_accepts_reasoning_field(
|
||||
key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None]
|
||||
key: str,
|
||||
value: object,
|
||||
values_by_field: Mapping[str, frozenset[str | None] | None],
|
||||
*,
|
||||
keeps_unlisted_keys: bool,
|
||||
) -> bool:
|
||||
if key == "effort":
|
||||
return True
|
||||
if key not in values_by_field:
|
||||
return False
|
||||
return keeps_unlisted_keys
|
||||
allowed: Final = values_by_field[key]
|
||||
return allowed is None or (isinstance(value, str | None) and value in allowed)
|
||||
|
||||
|
||||
def _reasoning_with_accepted_fields(
|
||||
reasoning: Mapping[str, object], values_by_field: Mapping[str, frozenset[str | None] | None]
|
||||
reasoning: Mapping[str, object],
|
||||
values_by_field: Mapping[str, frozenset[str | None] | None],
|
||||
*,
|
||||
keeps_unlisted_keys: bool = False,
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field)
|
||||
key: value
|
||||
for key, value in reasoning.items()
|
||||
if _openai_accepts_reasoning_field(key, value, values_by_field, keeps_unlisted_keys=keeps_unlisted_keys)
|
||||
}
|
||||
|
||||
|
||||
def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
|
||||
return _reasoning_with_accepted_fields(reasoning, _openai_reasoning_field_values(model))
|
||||
model_info: Final = _openai_model_info(model)
|
||||
if model_info is None:
|
||||
return openai_api_well_formed_reasoning(reasoning, keeps_unlisted_keys=True)
|
||||
if not model_info.get("supports_reasoning"):
|
||||
return _reasoning_with_accepted_fields(reasoning, MappingProxyType({}))
|
||||
return _reasoning_with_accepted_fields(reasoning, _openai_declared_reasoning_field_values(model_info))
|
||||
|
||||
|
||||
def openai_api_well_formed_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return _reasoning_with_accepted_fields(reasoning, _openai_declared_reasoning_field_values(None))
|
||||
def openai_api_well_formed_reasoning(
|
||||
reasoning: Mapping[str, object], *, keeps_unlisted_keys: bool
|
||||
) -> Mapping[str, object]:
|
||||
return _reasoning_with_accepted_fields(
|
||||
reasoning, _openai_declared_reasoning_field_values(None), keeps_unlisted_keys=keeps_unlisted_keys
|
||||
)
|
||||
|
||||
|
||||
def openai_well_formed_reasoning(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Final, cast
|
||||
|
|
@ -12,6 +13,7 @@ from openai.types.responses.response_compaction_item import ResponseCompactionIt
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig
|
||||
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
|
||||
|
|
@ -36,7 +38,6 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
from tests.unit.proxy.conftest import httpx_transport
|
||||
import time
|
||||
|
||||
_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$'
|
||||
|
||||
|
|
@ -3348,6 +3349,9 @@ class TestMalformedParamsFollowDropParams:
|
|||
pytest.param({"parallel_tool_calls": "true"}, id="parallel-tool-calls-string"),
|
||||
pytest.param({"previous_response_id": 12}, id="previous-response-id-int"),
|
||||
pytest.param({"previous_response_id": ""}, id="previous-response-id-empty"),
|
||||
pytest.param({"tool_choice": 5, "tools": [_WEATHER_TOOL]}, id="tool-choice-a-number-with-tools"),
|
||||
pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="tool-choice-a-list-with-tools"),
|
||||
pytest.param({"reasoning": "high"}, id="reasoning-not-an-object"),
|
||||
],
|
||||
)
|
||||
def test_a_malformed_value_is_dropped_under_drop_params_and_sent_as_is_without(
|
||||
|
|
@ -3419,7 +3423,6 @@ class TestMalformedParamsFollowDropParams:
|
|||
"params",
|
||||
[
|
||||
pytest.param({"tool_choice": {"name": "get_time"}, "tools": [_WEATHER_TOOL]}, id="choice-without-a-type"),
|
||||
pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="choice-not-an-object"),
|
||||
pytest.param(
|
||||
{"tool_choice": {"type": "function", "name": "get_time"}, "tools": _WEATHER_TOOL}, id="tools-not-a-list"
|
||||
),
|
||||
|
|
@ -3536,11 +3539,10 @@ class TestMalformedParamsFollowDropParams:
|
|||
[
|
||||
(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"),
|
||||
(AzureOpenAIOSeriesResponsesAPIConfig(), "o_series/gpt-6.1-sol"),
|
||||
(OpenAIResponsesAPIConfig(), "my-fine-tuned-model"),
|
||||
],
|
||||
ids=["azure", "azure-o-series", "outside-openai-catalog"],
|
||||
ids=["azure", "azure-o-series"],
|
||||
)
|
||||
def test_reasoning_values_no_openai_row_declares_are_dropped_where_the_openai_catalog_does_not_apply(
|
||||
def test_azure_drops_reasoning_values_and_keys_no_openai_row_declares(
|
||||
self, local_model_cost_map, monkeypatch, config, model
|
||||
):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
|
|
@ -3567,6 +3569,33 @@ class TestMalformedParamsFollowDropParams:
|
|||
assert kept["reasoning"] == declared
|
||||
assert sent["reasoning"] == malformed
|
||||
|
||||
def test_an_openai_model_outside_the_catalog_keeps_reasoning_keys_openai_does_not_define(
|
||||
self, local_model_cost_map, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
server_keys: Final = {"max_tokens": 2000, "exclude": True}
|
||||
requested: Final = {"effort": "high", "mode": "turbo", "context": "forever", "summary": "sometimes", **server_keys}
|
||||
|
||||
dropped: Final = OpenAIResponsesAPIConfig().map_openai_params(
|
||||
{"reasoning": dict(requested)}, "my-org/qwen3-32b", True
|
||||
)
|
||||
|
||||
assert dropped["reasoning"] == {"effort": "high", **server_keys}
|
||||
|
||||
def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map):
|
||||
row: Final = litellm.model_cost["gpt-6.1-sol"]
|
||||
new_mode: Final = "-".join(row["reasoning_modes"]) + "-next"
|
||||
request: Final = {"reasoning": {"effort": "high", "mode": new_mode}}
|
||||
|
||||
before_reload: Final = AzureOpenAIResponsesAPIConfig().map_openai_params(dict(request), "gpt-6.1-sol", True)
|
||||
adopt_model_cost_map(
|
||||
{**litellm.model_cost, "gpt-6.1-sol": {**row, "reasoning_modes": [*row["reasoning_modes"], new_mode]}}
|
||||
)
|
||||
after_reload: Final = AzureOpenAIResponsesAPIConfig().map_openai_params(dict(request), "gpt-6.1-sol", True)
|
||||
|
||||
assert before_reload["reasoning"] == {"effort": "high"}
|
||||
assert after_reload == request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_responses_litellm_router_no_metadata():
|
||||
|
|
|
|||
|
|
@ -1039,11 +1039,11 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr
|
|||
assert bridged(True) == {"effort": "high", "summary": "auto"}
|
||||
assert bridged(False) == requested
|
||||
|
||||
def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_outside_the_openai_catalog(
|
||||
def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_keeps_other_keys(
|
||||
local_model_cost_map,
|
||||
):
|
||||
declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1]
|
||||
requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "generate_summary": "auto"}
|
||||
requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "max_tokens": 2000}
|
||||
|
||||
def bridged(drop_params: bool) -> object:
|
||||
return openai_bridge_reasoning(
|
||||
|
|
@ -1055,7 +1055,7 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_outside_
|
|||
additional_drop_params=None,
|
||||
)
|
||||
|
||||
assert bridged(True) == {"effort": "high", "mode": declared_mode}
|
||||
assert bridged(True) == {"effort": "high", "mode": declared_mode, "max_tokens": 2000}
|
||||
assert bridged(False) == requested
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue