mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix: preserve caller thinking options and stop mutating optional params
Caller extra_body values now win over translated ones at every nesting level, and thinking.type translation keeps the caller's other thinking keys instead of rebuilding the dict from type and budget_tokens only. The translated extra_body is thawed into plain dicts before it reaches the request, since nested MappingProxyType values made completion() fail with 'Object of type mappingproxy is not JSON serializable'. get_optional_params now receives new passed_params / non_default_params dicts from the translation helper instead of having them rewritten in place Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
hx
parent
51b4da6f92
commit
755efe014d
3 changed files with 370 additions and 231 deletions
|
|
@ -50,16 +50,21 @@ def _thinking_enabled(thinking: object) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _thinking_type_candidate(thinking: object) -> str | None:
|
||||
match thinking:
|
||||
case bool():
|
||||
return "enabled" if thinking else "disabled"
|
||||
case str():
|
||||
return thinking
|
||||
case Mapping():
|
||||
raw: Final = thinking.get("type")
|
||||
return raw if isinstance(raw, str) else None
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _thinking_type_value(thinking: object, allowed: Sequence[str]) -> str | None:
|
||||
if isinstance(thinking, str):
|
||||
candidate = thinking
|
||||
elif isinstance(thinking, Mapping):
|
||||
raw = thinking.get("type")
|
||||
candidate = raw if isinstance(raw, str) else None
|
||||
elif isinstance(thinking, bool):
|
||||
candidate = "enabled" if thinking else "disabled"
|
||||
else:
|
||||
candidate = None
|
||||
candidate: Final = _thinking_type_candidate(thinking)
|
||||
if candidate is None:
|
||||
return None
|
||||
if not allowed or candidate in allowed:
|
||||
|
|
@ -72,10 +77,7 @@ def _thinking_type_value(thinking: object, allowed: Sequence[str]) -> str | None
|
|||
def _thinking_payload(thinking: object, typ: str) -> Mapping[str, object]:
|
||||
if not isinstance(thinking, Mapping):
|
||||
return MappingProxyType({"type": typ})
|
||||
budget: Final = thinking.get("budget_tokens")
|
||||
if isinstance(budget, int):
|
||||
return MappingProxyType({"type": typ, "budget_tokens": budget})
|
||||
return MappingProxyType({"type": typ})
|
||||
return MappingProxyType({**thinking, "type": typ})
|
||||
|
||||
|
||||
def _clamp_effort(value: object, allowed: Sequence[str]) -> str | None:
|
||||
|
|
@ -110,10 +112,19 @@ def _merged_mapping_value(left: Mapping[str, object], right: Mapping[str, object
|
|||
|
||||
|
||||
def _deep_merge_pair(left: Mapping[str, object], right: Mapping[str, object]) -> Mapping[str, object]:
|
||||
keys: Final = frozenset(left) | frozenset(right)
|
||||
keys: Final = (*left, *(key for key in right if key not in left))
|
||||
return MappingProxyType({key: _merged_mapping_value(left, right, key) for key in keys})
|
||||
|
||||
|
||||
def _thawed(value: object) -> object:
|
||||
mapping: Final = _mapping_or_none(value)
|
||||
return value if mapping is None else thaw_mapping(mapping)
|
||||
|
||||
|
||||
def thaw_mapping(mapping: Mapping[str, object]) -> dict[str, object]: # mutable-ok: JSON request body
|
||||
return {key: _thawed(item) for key, item in mapping.items()} # mutable-ok: JSON request body
|
||||
|
||||
|
||||
def _map_thinking_to_extra_body(
|
||||
*,
|
||||
thinking_param: str | None,
|
||||
|
|
@ -139,8 +150,6 @@ def _map_thinking_to_extra_body(
|
|||
return MappingProxyType(
|
||||
{"chat_template_kwargs": MappingProxyType({"enable_thinking": _thinking_enabled(thinking)})}
|
||||
)
|
||||
case None:
|
||||
return MappingProxyType({})
|
||||
case _:
|
||||
return MappingProxyType({})
|
||||
|
||||
|
|
@ -211,31 +220,9 @@ def translate_thinking_params(
|
|||
thinking_mapped: Final = any(key in patch for key in ("thinking", "enable_thinking", "chat_template_kwargs"))
|
||||
next_thinking: Final = thinking if (keep_thinking or not thinking_mapped) else None
|
||||
next_effort: Final = None if "reasoning_effort" in patch else effort
|
||||
merged_extra: Final = _deep_merge_pair(state.extra_body, patch)
|
||||
merged_extra: Final = _deep_merge_pair(patch, state.extra_body)
|
||||
return ThinkingParamsState(
|
||||
thinking=next_thinking,
|
||||
reasoning_effort=next_effort,
|
||||
extra_body=merged_extra,
|
||||
)
|
||||
|
||||
|
||||
def apply_thinking_param_translation(
|
||||
*,
|
||||
model_info: Mapping[str, object] | None,
|
||||
thinking: object | None,
|
||||
reasoning_effort: object | None,
|
||||
existing_extra_body: Mapping[str, object] | None,
|
||||
) -> ThinkingParamsState:
|
||||
base_extra: Final = (
|
||||
MappingProxyType({k: existing_extra_body[k] for k in existing_extra_body})
|
||||
if isinstance(existing_extra_body, Mapping)
|
||||
else MappingProxyType({})
|
||||
)
|
||||
return translate_thinking_params(
|
||||
model_info=model_info,
|
||||
state=ThinkingParamsState(
|
||||
thinking=thinking,
|
||||
reasoning_effort=reasoning_effort,
|
||||
extra_body=base_extra,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -295,7 +295,9 @@ from typing_extensions import assert_never
|
|||
|
||||
from litellm import utils as litellm_utils
|
||||
from litellm.litellm_core_utils.thinking_param_translation import (
|
||||
apply_thinking_param_translation,
|
||||
ThinkingParamsState,
|
||||
thaw_mapping,
|
||||
translate_thinking_params,
|
||||
)
|
||||
|
||||
# These are lazy loaded via __getattr__
|
||||
|
|
@ -4324,47 +4326,38 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict:
|
|||
return d
|
||||
|
||||
|
||||
def _apply_model_info_thinking_translation(
|
||||
def _translate_thinking_in_params(
|
||||
*,
|
||||
model_info: Mapping[str, object] | None,
|
||||
passed_params: dict,
|
||||
non_default_params: dict,
|
||||
) -> None:
|
||||
prior_thinking: Final = non_default_params.get("thinking")
|
||||
prior_effort: Final = non_default_params.get("reasoning_effort")
|
||||
existing_extra_raw: Final = passed_params.get("extra_body")
|
||||
existing_extra: Final = existing_extra_raw if isinstance(existing_extra_raw, Mapping) else None
|
||||
translated: Final = apply_thinking_param_translation(
|
||||
model_info=model_info,
|
||||
thinking=prior_thinking,
|
||||
reasoning_effort=prior_effort,
|
||||
existing_extra_body=existing_extra,
|
||||
passed_params: dict, # mutable-ok: get_optional_params hands over its legacy mutable params
|
||||
non_default_params: dict, # mutable-ok: get_optional_params hands over its legacy mutable params
|
||||
) -> tuple[dict, dict]: # mutable-ok: get_optional_params keeps mutating both copies downstream
|
||||
existing_extra_body: Final = passed_params.get("extra_body")
|
||||
state: Final = ThinkingParamsState(
|
||||
thinking=non_default_params.get("thinking"),
|
||||
reasoning_effort=non_default_params.get("reasoning_effort"),
|
||||
extra_body=existing_extra_body if isinstance(existing_extra_body, Mapping) else MappingProxyType({}),
|
||||
)
|
||||
translated: Final = translate_thinking_params(model_info=model_info, state=state)
|
||||
if translated is state:
|
||||
return passed_params, non_default_params
|
||||
thinking_values: Final = MappingProxyType(
|
||||
{"thinking": translated.thinking, "reasoning_effort": translated.reasoning_effort}
|
||||
)
|
||||
untouched_non_default: Final = MappingProxyType(
|
||||
{key: value for key, value in non_default_params.items() if key not in thinking_values}
|
||||
)
|
||||
surviving_thinking_values: Final = MappingProxyType(
|
||||
{key: value for key, value in thinking_values.items() if value is not None}
|
||||
)
|
||||
return (
|
||||
{ # mutable-ok: get_optional_params keeps mutating passed_params downstream
|
||||
**passed_params,
|
||||
**thinking_values,
|
||||
"extra_body": thaw_mapping(translated.extra_body),
|
||||
},
|
||||
{**untouched_non_default, **surviving_thinking_values}, # mutable-ok: _check_valid_arg pops unsupported keys
|
||||
)
|
||||
prior_extra: Final = dict(existing_extra) if existing_extra is not None else {} # mutable-ok: equality snapshot
|
||||
if (
|
||||
translated.thinking is prior_thinking
|
||||
and translated.reasoning_effort is prior_effort
|
||||
and dict(translated.extra_body) == prior_extra # mutable-ok: MappingProxyType equality snapshot
|
||||
):
|
||||
return
|
||||
|
||||
# mutable-ok: get_optional_params already mutates passed_params / non_default_params in place
|
||||
if translated.thinking is None:
|
||||
non_default_params.pop("thinking", None)
|
||||
passed_params["thinking"] = None
|
||||
else:
|
||||
non_default_params["thinking"] = translated.thinking
|
||||
passed_params["thinking"] = translated.thinking
|
||||
|
||||
if translated.reasoning_effort is None:
|
||||
non_default_params.pop("reasoning_effort", None)
|
||||
passed_params["reasoning_effort"] = None
|
||||
else:
|
||||
non_default_params["reasoning_effort"] = translated.reasoning_effort
|
||||
passed_params["reasoning_effort"] = translated.reasoning_effort
|
||||
|
||||
if translated.extra_body:
|
||||
passed_params["extra_body"] = dict(translated.extra_body) # mutable-ok: openai extra_body is a dict
|
||||
|
||||
|
||||
def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict:
|
||||
|
|
@ -4493,13 +4486,13 @@ def get_optional_params(
|
|||
**kwargs,
|
||||
):
|
||||
drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string
|
||||
passed_params: Final = locals().copy()
|
||||
special_params: Final = passed_params.pop("kwargs")
|
||||
untranslated_passed_params: Final = locals().copy()
|
||||
special_params: Final = untranslated_passed_params.pop("kwargs")
|
||||
# Remove base_model from passed_params so it doesn't interfere with
|
||||
# non_default_params / _check_valid_arg — it's a routing hint, not an
|
||||
# OpenAI param.
|
||||
passed_params.pop("base_model", None)
|
||||
model_info_for_translation: Final = passed_params.pop("model_info", None)
|
||||
untranslated_passed_params.pop("base_model", None)
|
||||
untranslated_passed_params.pop("model_info", None)
|
||||
provider_config: BaseConfig | None = None
|
||||
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]:
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
|
|
@ -4507,18 +4500,18 @@ def get_optional_params(
|
|||
provider=LlmProviders(custom_llm_provider),
|
||||
base_model=base_model,
|
||||
)
|
||||
non_default_params: Final = pre_process_non_default_params(
|
||||
passed_params=passed_params,
|
||||
untranslated_non_default_params: Final = pre_process_non_default_params(
|
||||
passed_params=untranslated_passed_params,
|
||||
special_params=special_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
additional_drop_params=additional_drop_params,
|
||||
model=model,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
_apply_model_info_thinking_translation(
|
||||
model_info=model_info_for_translation if isinstance(model_info_for_translation, Mapping) else None,
|
||||
passed_params=passed_params,
|
||||
non_default_params=non_default_params,
|
||||
passed_params, non_default_params = _translate_thinking_in_params(
|
||||
model_info=model_info,
|
||||
passed_params=untranslated_passed_params,
|
||||
non_default_params=untranslated_non_default_params,
|
||||
)
|
||||
optional_params = pre_process_optional_params(
|
||||
passed_params=passed_params,
|
||||
|
|
|
|||
|
|
@ -1,147 +1,279 @@
|
|||
import importlib
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.thinking_param_translation import (
|
||||
ThinkingParamsState,
|
||||
apply_thinking_param_translation,
|
||||
translate_thinking_params,
|
||||
)
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
|
||||
def _extra_body_model_info(**overrides: object) -> dict[str, object]:
|
||||
base: dict[str, object] = {
|
||||
_EXTRA_BODY_MODEL_INFO: Final = MappingProxyType(
|
||||
{
|
||||
"supports_reasoning": True,
|
||||
"thinking_param": "thinking.type",
|
||||
"thinking_values": ["enabled", "disabled"],
|
||||
"reasoning_effort_values": ["low", "high", "max"],
|
||||
"thinking_send_via": "extra_body",
|
||||
}
|
||||
return {**base, **overrides}
|
||||
)
|
||||
|
||||
|
||||
def test_translate_thinking_type_and_effort_to_extra_body():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(),
|
||||
thinking={"type": "enabled", "budget_tokens": 1024},
|
||||
reasoning_effort="high",
|
||||
existing_extra_body=None,
|
||||
)
|
||||
assert result.thinking is None
|
||||
assert result.reasoning_effort is None
|
||||
assert dict(result.extra_body) == {
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024},
|
||||
"reasoning_effort": "high",
|
||||
_CHAT_COMPLETION_RESPONSE: Final = MappingProxyType(
|
||||
{
|
||||
"id": "chatcmpl-thinking",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "deepseek-v4-flash",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_translate_enable_thinking_bool():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(
|
||||
thinking_param="enable_thinking",
|
||||
thinking_values=["true", "false"],
|
||||
def _model_info(**overrides: object) -> Mapping[str, object]:
|
||||
return MappingProxyType({**_EXTRA_BODY_MODEL_INFO, **overrides})
|
||||
|
||||
|
||||
def _state(
|
||||
*,
|
||||
thinking: object = None,
|
||||
reasoning_effort: object = None,
|
||||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> ThinkingParamsState:
|
||||
return ThinkingParamsState(thinking=thinking, reasoning_effort=reasoning_effort, extra_body=extra_body)
|
||||
|
||||
|
||||
def _as_plain(value: object) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
return {key: _as_plain(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _recording_client(bodies: list[object]) -> openai.OpenAI:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
bodies.append(json.loads(request.content))
|
||||
return httpx.Response(200, json=dict(_CHAT_COMPLETION_RESPONSE))
|
||||
|
||||
return openai.OpenAI(api_key="test-key", http_client=httpx.Client(transport=httpx.MockTransport(respond)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "thinking", "reasoning_effort", "expected_extra_body"),
|
||||
[
|
||||
pytest.param(
|
||||
_model_info(),
|
||||
{"type": "enabled", "budget_tokens": 1024, "clear_thinking": False},
|
||||
"high",
|
||||
{
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024, "clear_thinking": False},
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
id="thinking_type_keeps_caller_thinking_keys",
|
||||
),
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort=None,
|
||||
existing_extra_body=None,
|
||||
)
|
||||
assert result.thinking is None
|
||||
assert dict(result.extra_body) == {"enable_thinking": True}
|
||||
|
||||
|
||||
def test_translate_chat_template_kwargs():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(
|
||||
thinking_param="chat_template_kwargs",
|
||||
thinking_values=[],
|
||||
reasoning_effort_values=["low", "medium", "high"],
|
||||
pytest.param(
|
||||
_model_info(),
|
||||
{"type": "auto"},
|
||||
None,
|
||||
{"thinking": {"type": "enabled"}},
|
||||
id="thinking_type_auto_falls_back_to_enabled",
|
||||
),
|
||||
thinking={"type": "disabled"},
|
||||
reasoning_effort="medium",
|
||||
existing_extra_body=None,
|
||||
)
|
||||
assert dict(result.extra_body) == {
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
"reasoning_effort": "medium",
|
||||
}
|
||||
|
||||
|
||||
def test_translate_chat_template_kwargs_preserves_existing_nested_keys():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(
|
||||
thinking_param="chat_template_kwargs",
|
||||
thinking_values=[],
|
||||
pytest.param(
|
||||
_model_info(),
|
||||
False,
|
||||
None,
|
||||
{"thinking": {"type": "disabled"}},
|
||||
id="thinking_type_from_bool",
|
||||
),
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort=None,
|
||||
existing_extra_body={"chat_template_kwargs": {"reasoning_budget": 512}},
|
||||
)
|
||||
assert result.extra_body["chat_template_kwargs"]["reasoning_budget"] == 512
|
||||
assert result.extra_body["chat_template_kwargs"]["enable_thinking"] is True
|
||||
|
||||
|
||||
def test_translate_clamps_effort_aliases():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(reasoning_effort_values=["low", "high", "max"]),
|
||||
thinking=None,
|
||||
reasoning_effort="xhigh",
|
||||
existing_extra_body=None,
|
||||
)
|
||||
assert result.reasoning_effort is None
|
||||
assert result.extra_body["reasoning_effort"] == "max"
|
||||
|
||||
|
||||
def test_translate_provider_mapped_keeps_thinking_moves_effort():
|
||||
pytest.param(
|
||||
_model_info(),
|
||||
"enabled",
|
||||
None,
|
||||
{"thinking": {"type": "enabled"}},
|
||||
id="thinking_type_from_string",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="thinking", thinking_values=[]),
|
||||
{"type": "enabled", "budget_tokens": 2048},
|
||||
None,
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2048}},
|
||||
id="thinking_dict_passthrough",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="thinking", thinking_values=[]),
|
||||
True,
|
||||
None,
|
||||
{"thinking": {"type": "enabled"}},
|
||||
id="thinking_from_bool",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="enable_thinking"),
|
||||
{"type": "enabled"},
|
||||
None,
|
||||
{"enable_thinking": True},
|
||||
id="enable_thinking_from_type",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="enable_thinking"),
|
||||
{"enabled": True},
|
||||
None,
|
||||
{"enable_thinking": True},
|
||||
id="enable_thinking_from_enabled_flag",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="enable_thinking"),
|
||||
"true",
|
||||
None,
|
||||
{"enable_thinking": True},
|
||||
id="enable_thinking_from_string",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="enable_thinking"),
|
||||
False,
|
||||
None,
|
||||
{"enable_thinking": False},
|
||||
id="enable_thinking_from_bool",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="enable_thinking"),
|
||||
1,
|
||||
None,
|
||||
{"enable_thinking": False},
|
||||
id="enable_thinking_unrecognized_value_is_disabled",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="chat_template_kwargs", reasoning_effort_values=["low", "medium", "high"]),
|
||||
{"type": "disabled"},
|
||||
"medium",
|
||||
{"chat_template_kwargs": {"enable_thinking": False}, "reasoning_effort": "medium"},
|
||||
id="chat_template_kwargs",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(reasoning_effort_values=["low", "high", "max"]),
|
||||
None,
|
||||
"xhigh",
|
||||
{"reasoning_effort": "max"},
|
||||
id="effort_clamped_to_alias",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(reasoning_effort_values=["low"]),
|
||||
None,
|
||||
"minimal",
|
||||
{"reasoning_effort": "low"},
|
||||
id="effort_clamped_down",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(reasoning_effort_values=[]),
|
||||
None,
|
||||
"high",
|
||||
{"reasoning_effort": "high"},
|
||||
id="effort_passthrough_without_allowed_values",
|
||||
),
|
||||
pytest.param(
|
||||
_model_info(reasoning_effort_values=None),
|
||||
None,
|
||||
"high",
|
||||
{"reasoning_effort": "high"},
|
||||
id="effort_passthrough_when_allowed_values_missing",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_translate_moves_params_into_extra_body(
|
||||
model_info: Mapping[str, object],
|
||||
thinking: object,
|
||||
reasoning_effort: object,
|
||||
expected_extra_body: dict[str, object],
|
||||
):
|
||||
result = translate_thinking_params(
|
||||
model_info=_extra_body_model_info(thinking_send_via="provider_mapped"),
|
||||
state=ThinkingParamsState(
|
||||
model_info=model_info, state=_state(thinking=thinking, reasoning_effort=reasoning_effort)
|
||||
)
|
||||
|
||||
assert (result.thinking, result.reasoning_effort) == (None, None)
|
||||
assert _as_plain(result.extra_body) == expected_extra_body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "thinking", "reasoning_effort"),
|
||||
[
|
||||
pytest.param(None, {"type": "enabled"}, "high", id="no_model_info"),
|
||||
pytest.param(_model_info(thinking_send_via="n/a"), {"type": "enabled"}, "high", id="send_via_not_applicable"),
|
||||
pytest.param(_model_info(supports_reasoning=False), {"type": "enabled"}, "high", id="reasoning_unsupported"),
|
||||
pytest.param(_model_info(), None, None, id="nothing_requested"),
|
||||
pytest.param(_model_info(thinking_param="unknown"), {"type": "enabled"}, None, id="unknown_thinking_param"),
|
||||
pytest.param(_model_info(thinking_param=None), {"type": "enabled"}, None, id="thinking_param_missing"),
|
||||
pytest.param(_model_info(), {"type": "adaptive"}, None, id="thinking_type_not_allowed"),
|
||||
pytest.param(_model_info(), {"budget_tokens": 1024}, None, id="thinking_type_unreadable"),
|
||||
pytest.param(_model_info(), 1, None, id="thinking_type_unsupported_value"),
|
||||
pytest.param(
|
||||
_model_info(thinking_param="thinking", thinking_values=[]), "adaptive", None, id="thinking_value_unmapped"
|
||||
),
|
||||
pytest.param(_model_info(reasoning_effort_values=["low"]), None, "ultra", id="effort_without_fallback"),
|
||||
pytest.param(_model_info(), None, 5, id="effort_not_a_string"),
|
||||
],
|
||||
)
|
||||
def test_translate_returns_state_unchanged_when_nothing_applies(
|
||||
model_info: Mapping[str, object] | None,
|
||||
thinking: object,
|
||||
reasoning_effort: object,
|
||||
):
|
||||
state = _state(thinking=thinking, reasoning_effort=reasoning_effort)
|
||||
|
||||
assert translate_thinking_params(model_info=model_info, state=state) is state
|
||||
|
||||
|
||||
def test_translate_provider_mapped_keeps_thinking_and_moves_effort():
|
||||
thinking = {"type": "enabled"}
|
||||
|
||||
result = translate_thinking_params(
|
||||
model_info=_model_info(thinking_send_via="provider_mapped", supports_reasoning=False),
|
||||
state=_state(thinking=thinking, reasoning_effort="high"),
|
||||
)
|
||||
|
||||
assert result.thinking is thinking
|
||||
assert result.reasoning_effort is None
|
||||
assert _as_plain(result.extra_body) == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_translate_keeps_caller_extra_body_values_over_translated_ones():
|
||||
result = translate_thinking_params(
|
||||
model_info=_model_info(thinking_param="chat_template_kwargs", thinking_values=[]),
|
||||
state=_state(
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort="high",
|
||||
extra_body=MappingProxyType({}),
|
||||
extra_body={"chat_template_kwargs": {"enable_thinking": False, "reasoning_budget": 512}, "top_k": 20},
|
||||
),
|
||||
)
|
||||
assert result.thinking == {"type": "enabled"}
|
||||
assert result.reasoning_effort is None
|
||||
assert dict(result.extra_body) == {"reasoning_effort": "high"}
|
||||
|
||||
assert (result.thinking, result.reasoning_effort) == (None, None)
|
||||
assert _as_plain(result.extra_body) == {
|
||||
"chat_template_kwargs": {"enable_thinking": False, "reasoning_budget": 512},
|
||||
"reasoning_effort": "high",
|
||||
"top_k": 20,
|
||||
}
|
||||
|
||||
|
||||
def test_translate_noop_without_model_info():
|
||||
state = ThinkingParamsState(
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort="high",
|
||||
extra_body=MappingProxyType({}),
|
||||
)
|
||||
assert translate_thinking_params(model_info=None, state=state) is state
|
||||
|
||||
|
||||
def test_translate_noop_when_send_via_na():
|
||||
result = apply_thinking_param_translation(
|
||||
model_info=_extra_body_model_info(thinking_send_via="n/a"),
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort="high",
|
||||
existing_extra_body=None,
|
||||
)
|
||||
assert result.thinking == {"type": "enabled"}
|
||||
assert result.reasoning_effort == "high"
|
||||
assert dict(result.extra_body) == {}
|
||||
|
||||
|
||||
def test_get_optional_params_openai_drop_translates_via_model_info():
|
||||
def test_get_optional_params_moves_thinking_into_extra_body():
|
||||
optional_params = get_optional_params(
|
||||
model="deepseek-v4-flash",
|
||||
custom_llm_provider="openai",
|
||||
drop_params=True,
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort="high",
|
||||
model_info=_extra_body_model_info(),
|
||||
model_info=_model_info(),
|
||||
)
|
||||
assert optional_params.get("thinking") is None
|
||||
assert optional_params.get("reasoning_effort") is None
|
||||
assert optional_params["extra_body"]["thinking"] == {"type": "enabled"}
|
||||
assert optional_params["extra_body"]["reasoning_effort"] == "high"
|
||||
|
||||
assert "thinking" not in optional_params
|
||||
assert "reasoning_effort" not in optional_params
|
||||
assert optional_params["extra_body"] == {"thinking": {"type": "enabled"}, "reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_get_optional_params_openai_drop_without_model_info_drops_params():
|
||||
def test_get_optional_params_without_model_info_drops_thinking():
|
||||
optional_params = get_optional_params(
|
||||
model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -149,11 +281,9 @@ def test_get_optional_params_openai_drop_without_model_info_drops_params():
|
|||
thinking={"type": "enabled"},
|
||||
reasoning_effort="high",
|
||||
)
|
||||
extra_body = optional_params.get("extra_body") or {}
|
||||
assert "thinking" not in extra_body
|
||||
assert "reasoning_effort" not in extra_body
|
||||
assert optional_params.get("thinking") is None
|
||||
assert optional_params.get("reasoning_effort") is None
|
||||
|
||||
assert "thinking" not in optional_params
|
||||
assert "thinking" not in (optional_params.get("extra_body") or {})
|
||||
|
||||
|
||||
def test_get_optional_params_does_not_reintroduce_dropped_thinking():
|
||||
|
|
@ -163,39 +293,68 @@ def test_get_optional_params_does_not_reintroduce_dropped_thinking():
|
|||
drop_params=True,
|
||||
thinking={"type": "enabled"},
|
||||
additional_drop_params=["thinking"],
|
||||
model_info=_extra_body_model_info(
|
||||
thinking_param="chat_template_kwargs",
|
||||
thinking_values=[],
|
||||
),
|
||||
model_info=_model_info(thinking_param="chat_template_kwargs", thinking_values=[]),
|
||||
)
|
||||
extra_body = optional_params.get("extra_body") or {}
|
||||
assert optional_params.get("thinking") is None
|
||||
assert "enable_thinking" not in extra_body
|
||||
assert "chat_template_kwargs" not in extra_body
|
||||
|
||||
assert "thinking" not in optional_params
|
||||
assert optional_params.get("extra_body") in (None, {})
|
||||
|
||||
|
||||
def test_batch_completion_vllm_passes_model_info(monkeypatch):
|
||||
batch_completion_mod = importlib.import_module("litellm.batch_completion.main")
|
||||
def test_get_optional_params_leaves_caller_extra_body_untouched_and_serializable():
|
||||
caller_extra_body = {"chat_template_kwargs": {"reasoning_budget": 512}}
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
looked_up: dict[str, object] = _extra_body_model_info(thinking_param="enable_thinking")
|
||||
|
||||
def fake_get_optional_params(**kwargs: object) -> dict[str, object]:
|
||||
captured.update(kwargs)
|
||||
return {}
|
||||
|
||||
def fake_batch_completions(**kwargs: object) -> list[str]:
|
||||
return ["ok"]
|
||||
|
||||
def fake_get_model_info(**kwargs: object) -> dict[str, object]:
|
||||
return looked_up
|
||||
|
||||
monkeypatch.setattr(batch_completion_mod, "get_optional_params", fake_get_optional_params)
|
||||
monkeypatch.setattr(batch_completion_mod.vllm_handler, "batch_completions", fake_batch_completions)
|
||||
monkeypatch.setattr(batch_completion_mod, "get_model_info", fake_get_model_info)
|
||||
|
||||
batch_completion_mod.batch_completion(
|
||||
model="vllm/some-model",
|
||||
messages=[[{"role": "user", "content": "hi"}]],
|
||||
optional_params = get_optional_params(
|
||||
model="deepseek-v4-flash",
|
||||
custom_llm_provider="openai",
|
||||
thinking={"type": "enabled"},
|
||||
extra_body=caller_extra_body,
|
||||
model_info=_model_info(thinking_param="chat_template_kwargs", thinking_values=[]),
|
||||
)
|
||||
assert captured.get("model_info") == looked_up
|
||||
|
||||
expected_extra_body = {"chat_template_kwargs": {"reasoning_budget": 512, "enable_thinking": True}}
|
||||
assert optional_params["extra_body"] == expected_extra_body
|
||||
assert json.loads(json.dumps(optional_params["extra_body"])) == expected_extra_body
|
||||
assert copy.deepcopy(optional_params)["extra_body"] == expected_extra_body
|
||||
assert caller_extra_body == {"chat_template_kwargs": {"reasoning_budget": 512}}
|
||||
|
||||
|
||||
def test_completion_sends_translated_thinking_on_the_wire():
|
||||
bodies: list[object] = []
|
||||
|
||||
litellm.completion(
|
||||
model="openai/deepseek-v4-flash",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
thinking={"type": "enabled", "budget_tokens": 1024},
|
||||
reasoning_effort="high",
|
||||
model_info=dict(_model_info()),
|
||||
client=_recording_client(bodies),
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
assert bodies == [
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024},
|
||||
"reasoning_effort": "high",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_batch_completion_translates_every_request_like_completion():
|
||||
bodies: list[object] = []
|
||||
|
||||
litellm.batch_completion(
|
||||
model="openai/deepseek-v4-flash",
|
||||
messages=[[{"role": "user", "content": "one"}], [{"role": "user", "content": "two"}]],
|
||||
thinking={"type": "enabled"},
|
||||
model_info=dict(_model_info(thinking_param="enable_thinking")),
|
||||
client=_recording_client(bodies),
|
||||
num_retries=0,
|
||||
max_workers=1,
|
||||
)
|
||||
|
||||
assert sorted(bodies, key=lambda body: json.dumps(body, sort_keys=True)) == [
|
||||
{"model": "deepseek-v4-flash", "messages": [{"role": "user", "content": "one"}], "enable_thinking": True},
|
||||
{"model": "deepseek-v4-flash", "messages": [{"role": "user", "content": "two"}], "enable_thinking": True},
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue