mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
fix(anthropic): translate raw adaptive thinking for chat completions on pre-4.6 models
Clients that pass thinking={"type": "adaptive"} directly (not via the
reasoning_effort alias) on the /chat/completions interface had it forwarded
unmodified to pre-4.6 Anthropic models, which reject the shape. Mirrors the
translation already applied on the native /v1/messages passthrough (#32867):
translate to legacy thinking={type: enabled, budget_tokens}, capped below
max_tokens, dropping thinking when max_tokens can't fit even the minimum
budget. Hoists the shared budget-capping helper onto AnthropicConfig so both
paths use one implementation.
This commit is contained in:
parent
92dfbdbb21
commit
831dbbc4df
3 changed files with 126 additions and 18 deletions
|
|
@ -227,6 +227,10 @@ DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING = (
|
|||
"Sonnet 4.6+, and Mythos Preview."
|
||||
)
|
||||
|
||||
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING = (
|
||||
"Dropping adaptive `thinking` for model=%s: max_tokens is too small to fit the minimum thinking budget."
|
||||
)
|
||||
|
||||
DROP_UNSUPPORTED_SPEED_WARNING = (
|
||||
"Dropping unsupported `speed` for model=%s (drop_params=True). Fast mode is only supported on select Opus models."
|
||||
)
|
||||
|
|
@ -1220,6 +1224,23 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cap_thinking_budget_to_max_tokens(
|
||||
thinking: AnthropicThinkingParam, max_tokens: Optional[int]
|
||||
) -> Optional[AnthropicThinkingParam]:
|
||||
"""Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
|
||||
requires ``max_tokens > budget_tokens``). Returns the (possibly capped)
|
||||
thinking dict, or ``None`` when ``max_tokens`` is too small to fit even the
|
||||
minimum thinking budget and thinking should be dropped."""
|
||||
budget = thinking.get("budget_tokens")
|
||||
if max_tokens is None or not isinstance(budget, int):
|
||||
return thinking
|
||||
if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS:
|
||||
return None
|
||||
if budget < max_tokens:
|
||||
return thinking
|
||||
return AnthropicThinkingParam(type=thinking.get("type", "enabled"), budget_tokens=max_tokens - 1)
|
||||
|
||||
def _extract_json_schema_from_response_format(self, value: Optional[dict]) -> Optional[dict]:
|
||||
if value is None:
|
||||
return None
|
||||
|
|
@ -1463,7 +1484,37 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
):
|
||||
optional_params["metadata"] = {"user_id": value}
|
||||
elif param == "thinking":
|
||||
optional_params["thinking"] = value
|
||||
if (
|
||||
isinstance(value, dict)
|
||||
and value.get("type") == "adaptive"
|
||||
and not AnthropicConfig._is_adaptive_thinking_model(model)
|
||||
):
|
||||
# Callers (e.g. Claude Code) send adaptive thinking
|
||||
# unconditionally; translate it down to the legacy
|
||||
# `thinking={type: enabled, budget_tokens}` interface a
|
||||
# pre-4.6 model actually supports instead of forwarding a
|
||||
# shape the model will reject.
|
||||
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
|
||||
legacy_thinking = AnthropicConfig._map_reasoning_effort(
|
||||
reasoning_effort="medium",
|
||||
model=model,
|
||||
llm_provider=self.custom_llm_provider or "anthropic",
|
||||
)
|
||||
capped_thinking = (
|
||||
AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||
if legacy_thinking is not None
|
||||
else None
|
||||
)
|
||||
if capped_thinking is not None:
|
||||
optional_params["thinking"] = capped_thinking
|
||||
else:
|
||||
litellm.verbose_logger.warning(
|
||||
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
|
||||
model,
|
||||
)
|
||||
optional_params.pop("thinking", None)
|
||||
else:
|
||||
optional_params["thinking"] = value
|
||||
elif param == "reasoning_effort":
|
||||
# Accept both string ("low") and dict ({"effort": "low",
|
||||
# "summary": "concise"}). The Responses->Chat parser keeps the
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
|
|||
import httpx
|
||||
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
|
|
@ -353,7 +352,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
except _BadRequestError as e:
|
||||
raise AnthropicError(message=str(e.message), status_code=400)
|
||||
capped_thinking = (
|
||||
AnthropicMessagesConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||
AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||
if legacy_thinking is not None
|
||||
else None
|
||||
)
|
||||
|
|
@ -371,21 +370,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
else:
|
||||
optional_params.pop("output_config", None)
|
||||
|
||||
@staticmethod
|
||||
def _cap_thinking_budget_to_max_tokens(thinking: Dict, max_tokens: Optional[int]) -> Optional[Dict]:
|
||||
"""Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
|
||||
requires ``max_tokens > budget_tokens``). Returns the (possibly capped)
|
||||
thinking dict, or ``None`` when ``max_tokens`` is too small to fit even the
|
||||
minimum thinking budget and thinking should be dropped."""
|
||||
budget = thinking.get("budget_tokens")
|
||||
if max_tokens is None or not isinstance(budget, int):
|
||||
return thinking
|
||||
if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS:
|
||||
return None
|
||||
if budget < max_tokens:
|
||||
return thinking
|
||||
return {**thinking, "budget_tokens": max_tokens - 1}
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET,
|
||||
|
|
@ -2443,6 +2444,78 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models():
|
|||
assert result["output_config"]["effort"] == effort_map[effort]
|
||||
|
||||
|
||||
def test_raw_adaptive_thinking_translates_to_legacy_for_pre_46_model():
|
||||
"""Clients like Claude Code send ``thinking={"type": "adaptive"}`` directly
|
||||
(not via ``reasoning_effort``) on every request, regardless of which model
|
||||
the request routes to. For a pre-4.6 model that doesn't understand
|
||||
adaptive thinking, this must be translated to the legacy
|
||||
``thinking={type: enabled, budget_tokens}`` interface instead of being
|
||||
forwarded raw, which Anthropic would reject."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"thinking": {"type": "adaptive"}, "max_tokens": 8192},
|
||||
optional_params={},
|
||||
model="claude-haiku-4-5-20251001",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["thinking"]["type"] == "enabled"
|
||||
assert result["thinking"]["budget_tokens"] == DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET
|
||||
|
||||
|
||||
def test_raw_adaptive_thinking_budget_capped_below_max_tokens():
|
||||
"""Anthropic requires ``max_tokens > thinking.budget_tokens``. When the
|
||||
default medium budget wouldn't fit, it must be capped below max_tokens
|
||||
rather than forwarded as an invalid combination."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
max_tokens = DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET - 100
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"thinking": {"type": "adaptive"}, "max_tokens": max_tokens},
|
||||
optional_params={},
|
||||
model="claude-haiku-4-5-20251001",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["thinking"]["type"] == "enabled"
|
||||
assert result["thinking"]["budget_tokens"] == max_tokens - 1
|
||||
|
||||
|
||||
def test_raw_adaptive_thinking_dropped_when_max_tokens_too_small():
|
||||
"""When max_tokens can't fit even the minimum thinking budget, thinking
|
||||
must be dropped entirely so the request still succeeds, matching how the
|
||||
native /v1/messages passthrough already handles this."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
result = config.map_openai_params(
|
||||
non_default_params={
|
||||
"thinking": {"type": "adaptive"},
|
||||
"max_tokens": ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
},
|
||||
optional_params={},
|
||||
model="claude-haiku-4-5-20251001",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "thinking" not in result
|
||||
|
||||
|
||||
def test_raw_adaptive_thinking_untouched_for_46_plus_model():
|
||||
"""Adaptive-thinking models understand ``thinking={"type": "adaptive"}``
|
||||
natively, so it must pass through unmodified."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"thinking": {"type": "adaptive"}, "max_tokens": 8192},
|
||||
optional_params={},
|
||||
model="claude-sonnet-4-6-20260219",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["thinking"] == {"type": "adaptive"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch):
|
||||
original_model_cost = litellm.model_cost
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue