mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-01 02:02:20 +00:00
fix(anthropic): cap reasoning_effort thinking budget below max_tokens on /v1/messages
A deployment carrying reasoning_effort in its litellm_params on the /v1/messages passthrough mapped the effort to a legacy thinking block whose budget_tokens was forwarded as is, so any request whose max_tokens sat at or below that budget was rejected upstream with a 400. The mapped budget now runs through the same cap the adaptive-to-legacy branch and the chat path already use: it is clamped to max_tokens - 1, and dropped with a warning when even the minimum budget cannot fit. The cap helper becomes public since three call sites outside AnthropicConfig use it.
This commit is contained in:
parent
5c034fda74
commit
71a951691a
6 changed files with 66 additions and 10 deletions
|
|
@ -1268,7 +1268,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _cap_thinking_budget_to_max_tokens(
|
def cap_thinking_budget_to_max_tokens(
|
||||||
thinking: AnthropicThinkingParam, max_tokens: int | None
|
thinking: AnthropicThinkingParam, max_tokens: int | None
|
||||||
) -> AnthropicThinkingParam | None:
|
) -> AnthropicThinkingParam | None:
|
||||||
"""Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
|
"""Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
|
||||||
|
|
@ -1530,7 +1530,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
||||||
llm_provider=self._resolved_provider,
|
llm_provider=self._resolved_provider,
|
||||||
)
|
)
|
||||||
capped_thinking = (
|
capped_thinking = (
|
||||||
AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||||
if legacy_thinking is not None
|
if legacy_thinking is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,11 @@ DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING: Final = (
|
||||||
"minimum thinking budget."
|
"minimum thinking budget."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
|
||||||
|
"Dropping `thinking` mapped from reasoning_effort=%s for model=%s: max_tokens=%s "
|
||||||
|
"is too small to fit the minimum thinking budget."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||||
@property
|
@property
|
||||||
|
|
@ -335,11 +340,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||||
return headers, api_base
|
return headers, api_base
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _translate_reasoning_effort_to_anthropic(model: str, optional_params: dict, custom_llm_provider: str) -> None:
|
def _translate_reasoning_effort_to_anthropic(
|
||||||
|
model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
|
||||||
|
) -> None:
|
||||||
"""Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
|
"""Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
|
||||||
|
|
||||||
Caller-supplied ``thinking`` / ``output_config`` win over the alias.
|
Caller-supplied ``thinking`` / ``output_config`` win over the alias.
|
||||||
``effort='none'`` clears both. Invalid efforts raise a 400.
|
``effort='none'`` clears both. Invalid efforts raise a 400. A mapped
|
||||||
|
thinking budget is capped below ``max_tokens`` and dropped when even
|
||||||
|
the minimum budget cannot fit.
|
||||||
"""
|
"""
|
||||||
from litellm.exceptions import BadRequestError as _BadRequestError
|
from litellm.exceptions import BadRequestError as _BadRequestError
|
||||||
from litellm.llms.anthropic.chat.transformation import (
|
from litellm.llms.anthropic.chat.transformation import (
|
||||||
|
|
@ -365,7 +374,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||||
optional_params.pop("output_config", None)
|
optional_params.pop("output_config", None)
|
||||||
return
|
return
|
||||||
|
|
||||||
optional_params.setdefault("thinking", mapped_thinking)
|
fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens)
|
||||||
|
if fitted_thinking is None:
|
||||||
|
verbose_logger.warning(DROP_UNFITTING_REASONING_EFFORT_WARNING, reasoning_effort, model, max_tokens)
|
||||||
|
return
|
||||||
|
|
||||||
|
optional_params.setdefault("thinking", fitted_thinking)
|
||||||
if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
|
if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
|
||||||
mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
|
mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
|
||||||
if mapped_effort is None:
|
if mapped_effort is None:
|
||||||
|
|
@ -510,7 +524,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||||
except _BadRequestError as e:
|
except _BadRequestError as e:
|
||||||
raise AnthropicError(message=str(e.message), status_code=400)
|
raise AnthropicError(message=str(e.message), status_code=400)
|
||||||
capped_thinking: Final = (
|
capped_thinking: Final = (
|
||||||
AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||||
if legacy_thinking is not None
|
if legacy_thinking is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
@ -582,6 +596,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||||
self._translate_reasoning_effort_to_anthropic(
|
self._translate_reasoning_effort_to_anthropic(
|
||||||
model=model,
|
model=model,
|
||||||
optional_params=anthropic_messages_optional_request_params,
|
optional_params=anthropic_messages_optional_request_params,
|
||||||
|
max_tokens=max_tokens,
|
||||||
custom_llm_provider=self._resolved_provider,
|
custom_llm_provider=self._resolved_provider,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -924,7 +924,7 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
custom_llm_provider="bedrock",
|
custom_llm_provider="bedrock",
|
||||||
)
|
)
|
||||||
capped = (
|
capped = (
|
||||||
AnthropicConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
AnthropicConfig.cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
|
||||||
if legacy_thinking is not None
|
if legacy_thinking is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,10 @@ from litellm.llms.anthropic.common_utils import AnthropicError
|
||||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||||
AnthropicMessagesConfig,
|
AnthropicMessagesConfig,
|
||||||
)
|
)
|
||||||
|
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
|
||||||
|
from litellm.llms.openai_like.messages.transformation import (
|
||||||
|
JSONProviderAnthropicMessagesConfig,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _claude_code_payload(effort="medium", max_tokens=8192, **output_config_extra):
|
def _claude_code_payload(effort="medium", max_tokens=8192, **output_config_extra):
|
||||||
|
|
@ -294,3 +298,39 @@ def test_non_adaptive_request_without_effort_is_untouched():
|
||||||
|
|
||||||
assert "thinking" not in result
|
assert "thinking" not in result
|
||||||
assert "output_config" not in result
|
assert "output_config" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_reasoning_effort_budget_capped_below_max_tokens():
|
||||||
|
result = _transform("claude-haiku-4-5", {"max_tokens": 4000, "reasoning_effort": "xhigh"})
|
||||||
|
|
||||||
|
assert result["thinking"] == {"type": "enabled", "budget_tokens": 3999}
|
||||||
|
assert result["max_tokens"] == 4000
|
||||||
|
|
||||||
|
|
||||||
|
def test_reasoning_effort_thinking_dropped_when_min_budget_cannot_fit():
|
||||||
|
result = _transform("claude-haiku-4-5", {"max_tokens": 1024, "reasoning_effort": "xhigh"})
|
||||||
|
|
||||||
|
assert "thinking" not in result
|
||||||
|
assert result["max_tokens"] == 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_reasoning_effort_budget_capped_for_openai_like_messages_upstream():
|
||||||
|
provider = SimpleProviderConfig(
|
||||||
|
"meta",
|
||||||
|
{
|
||||||
|
"base_url": "https://api.meta.ai/v1",
|
||||||
|
"api_key_env": "META_API_KEY",
|
||||||
|
"supported_endpoints": ["/v1/messages"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = JSONProviderAnthropicMessagesConfig(provider).transform_anthropic_messages_request(
|
||||||
|
model="muse-spark-1.2",
|
||||||
|
messages=[{"role": "user", "content": "Hello"}],
|
||||||
|
anthropic_messages_optional_request_params={"max_tokens": 4000, "reasoning_effort": "xhigh"},
|
||||||
|
litellm_params={},
|
||||||
|
headers={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["thinking"] == {"type": "enabled", "budget_tokens": 3999}
|
||||||
|
assert result["max_tokens"] == 4000
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ def test_reasoning_effort_none_clears_thinking_and_output_config():
|
||||||
|
|
||||||
def test_reasoning_effort_on_non_adaptive_model_uses_thinking_budget():
|
def test_reasoning_effort_on_non_adaptive_model_uses_thinking_budget():
|
||||||
config = AnthropicMessagesConfig()
|
config = AnthropicMessagesConfig()
|
||||||
optional_params = {"max_tokens": 1024, "reasoning_effort": "high"}
|
optional_params = {"max_tokens": 8192, "reasoning_effort": "high"}
|
||||||
|
|
||||||
result = config.transform_anthropic_messages_request(
|
result = config.transform_anthropic_messages_request(
|
||||||
model="claude-opus-4-5",
|
model="claude-opus-4-5",
|
||||||
|
|
@ -86,7 +86,7 @@ def test_reasoning_effort_on_non_adaptive_model_uses_thinking_budget():
|
||||||
assert isinstance(thinking, dict)
|
assert isinstance(thinking, dict)
|
||||||
assert thinking.get("type") == "enabled"
|
assert thinking.get("type") == "enabled"
|
||||||
assert isinstance(thinking.get("budget_tokens"), int)
|
assert isinstance(thinking.get("budget_tokens"), int)
|
||||||
assert thinking["budget_tokens"] >= 1024
|
assert 1024 <= thinking["budget_tokens"] < result["max_tokens"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("bad_effort", ["invalid", "disabled", ""])
|
@pytest.mark.parametrize("bad_effort", ["invalid", "disabled", ""])
|
||||||
|
|
|
||||||
|
|
@ -254,7 +254,7 @@ def test_request_maps_reasoning_effort_to_thinking(config):
|
||||||
model="claude-sonnet-4-20250514",
|
model="claude-sonnet-4-20250514",
|
||||||
messages=[{"role": "user", "content": "hi"}],
|
messages=[{"role": "user", "content": "hi"}],
|
||||||
anthropic_messages_optional_request_params={
|
anthropic_messages_optional_request_params={
|
||||||
"max_tokens": 1024,
|
"max_tokens": 8192,
|
||||||
"reasoning_effort": "medium",
|
"reasoning_effort": "medium",
|
||||||
},
|
},
|
||||||
litellm_params=GenericLiteLLMParams(),
|
litellm_params=GenericLiteLLMParams(),
|
||||||
|
|
@ -264,6 +264,7 @@ def test_request_maps_reasoning_effort_to_thinking(config):
|
||||||
assert "reasoning_effort" not in payload
|
assert "reasoning_effort" not in payload
|
||||||
assert isinstance(payload.get("thinking"), dict)
|
assert isinstance(payload.get("thinking"), dict)
|
||||||
assert payload["thinking"].get("type") == "enabled"
|
assert payload["thinking"].get("type") == "enabled"
|
||||||
|
assert payload["thinking"]["budget_tokens"] < payload["max_tokens"]
|
||||||
|
|
||||||
|
|
||||||
def test_passthrough_disables_anthropic_beta_filtering(config):
|
def test_passthrough_disables_anthropic_beta_filtering(config):
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue