scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-07 07:58:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 150

fix(anthropic): pass max_tokens to the Mantle clear_thinking check and keep the edit on unmapped Claude 5 ids

Browse source 
Bedrock Mantle now drops the clear_thinking edit when max_tokens is too small
for it to add thinking back, instead of sending the edit without thinking.

A cost-map fallback rule marks every Claude model at major version 5 or higher
as thinking_on_by_default, so new minors and dated ids keep the edit.
This commit is contained in:
mateo-berri 2026-10-30 08:32:55 -07:00
parent 686af04f1e
commit f353aabb2a
10 changed files with 86 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -118,6 +118,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: MutableMapping[str, object], # mutable-ok: edits optional params in place
model: str,
custom_llm_provider: str,
max_tokens: int | None,
) -> None:
optional_params.pop("thinking", None)
drop_clear_thinking_edits_for_model(optional_params, model, custom_llm_provider)
@ -501,7 +502,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raise AnthropicError(message=str(e.message), status_code=400)
if mapped_thinking is None:
self._drop_thinking(optional_params, model, custom_llm_provider)
self._drop_thinking(optional_params, model, custom_llm_provider, max_tokens)
optional_params.pop("output_config", None)
return
@ -509,7 +510,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if fitted_thinking is None:
verbose_logger.warning(DROP_UNFITTING_REASONING_EFFORT_WARNING, reasoning_effort, model, max_tokens)
if "thinking" not in optional_params:
self._drop_thinking(optional_params, model, custom_llm_provider)
self._drop_thinking(optional_params, model, custom_llm_provider, max_tokens)
return
optional_params.setdefault("thinking", fitted_thinking)
@ -597,7 +598,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
or AnthropicConfig.validate_effort_for_model(model, effort, custom_llm_provider) is None
):
if adaptive_thinking:
self._drop_thinking(optional_params, model, custom_llm_provider)
self._drop_thinking(optional_params, model, custom_llm_provider, max_tokens)
return
supports_thinking: Final = AnthropicModelInfo.supports_model_capability(
@ -625,7 +626,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params["thinking"] = capped_thinking
else:
verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING, model)
self._drop_thinking(optional_params, model, custom_llm_provider)
self._drop_thinking(optional_params, model, custom_llm_provider, max_tokens)
if isinstance(output_config, dict) and "effort" in output_config:
residual: Final = {k: v for k, v in output_config.items() if k != "effort"}

1
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py
View file

@ -105,6 +105,7 @@ class AmazonAnthropicClaudeMessagesConfig(
optional_params: MutableMapping[str, object], # mutable-ok: edits optional params in place
model: str,
custom_llm_provider: str,
max_tokens: int | None,
) -> None:
"""Keep the clear_thinking edit: it makes
``_ensure_thinking_for_clear_thinking_context_management`` add thinking back,

12
litellm/llms/bedrock_mantle/messages/transformation.py
View file

@ -51,14 +51,18 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM
optional_params: MutableMapping[str, object], # mutable-ok: edits optional params in place
model: str,
custom_llm_provider: str,
max_tokens: int | None,
) -> None:
super()._drop_thinking(optional_params, model, custom_llm_provider)
if not self._adds_thinking_back_for_clear_thinking(optional_params, model):
super()._drop_thinking(optional_params, model, custom_llm_provider, max_tokens)
if not self._adds_thinking_back_for_clear_thinking(optional_params, model, max_tokens):
drop_clear_thinking_edits_for_model(optional_params, model, custom_llm_provider)
def _adds_thinking_back_for_clear_thinking(self, optional_params: Mapping[str, object], model: str) -> bool:
def _adds_thinking_back_for_clear_thinking(
self, optional_params: Mapping[str, object], model: str, max_tokens: int | None
) -> bool:
trial_request: Final = {
key: optional_params[key] for key in ("context_management", "max_tokens") if key in optional_params
"context_management": optional_params.get("context_management"),
"max_tokens": max_tokens,
}
return self._ensure_thinking_for_clear_thinking_context_management(trial_request, model, drop_params=False)

8
litellm/model_prices_and_context_window_backup.json
View file

@ -63583,6 +63583,14 @@
"thinking_always_on": true
}
},
{
"name": "claude-thinking-on-by-default",
"pattern": "claude-[a-z]+-[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?",
"description": "Claude at major version 5 or higher, in any id shape that contains claude-<family>-<major>, so new minors such as claude-sonnet-5-1 and provider-namespaced ids match. Every Claude 5 model thinks when a request carries no thinking param, so a clear_thinking context-management edit stays valid when LiteLLM drops thinking. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41.",
"model_info": {
"thinking_on_by_default": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",

8
model_prices_and_context_window.json
View file

@ -63583,6 +63583,14 @@
"thinking_always_on": true
}
},
{
"name": "claude-thinking-on-by-default",
"pattern": "claude-[a-z]+-[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?",
"description": "Claude at major version 5 or higher, in any id shape that contains claude-<family>-<major>, so new minors such as claude-sonnet-5-1 and provider-namespaced ids match. Every Claude 5 model thinks when a request carries no thinking param, so a clear_thinking context-management edit stays valid when LiteLLM drops thinking. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41.",
"model_info": {
"thinking_on_by_default": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",

18
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -9838,11 +9838,11 @@ def _edits_for(keeps_thinking: bool) -> list[dict[str, str]]:
)
@pytest.mark.parametrize("context_management_first", [True, False])
def test_map_openai_params_drops_clear_thinking_edit_exactly_when_thinking_is(
model, params, keeps_thinking, context_management_first
):
model: str, params: dict[str, object], keeps_thinking: bool, context_management_first: bool
) -> None:
context_management: Final = {"context_management": {"edits": [CLEAR_THINKING_EDIT, CLEAR_TOOL_USES_EDIT]}}
rest: Final = {"max_tokens": 8192, **params}
result = AnthropicConfig().map_openai_params(
result: Final = AnthropicConfig().map_openai_params(
non_default_params={**context_management, **rest}
if context_management_first
else {**rest, **context_management},
@ -9862,8 +9862,10 @@ def test_map_openai_params_drops_clear_thinking_edit_exactly_when_thinking_is(
pytest.param("claude-sonnet-4-5", {"summary": "concise"}, id="effort-less-dict"),
],
)
def test_map_openai_params_keeps_clear_thinking_edit_when_thinking_was_not_dropped(model, reasoning_effort):
result = AnthropicConfig().map_openai_params(
def test_map_openai_params_keeps_clear_thinking_edit_when_thinking_was_not_dropped(
model: str, reasoning_effort: str | dict[str, str]
) -> None:
result: Final = AnthropicConfig().map_openai_params(
non_default_params={
"context_management": {"edits": [CLEAR_THINKING_EDIT]},
"max_tokens": 8192,
@ -9886,10 +9888,10 @@ def test_map_openai_params_keeps_clear_thinking_edit_when_thinking_was_not_dropp
],
)
def test_transform_request_drops_clear_thinking_edit_with_thinking_for_tool_turn_without_thinking_blocks(
monkeypatch, model, modify_params, keeps_thinking, keeps_edit
):
monkeypatch: pytest.MonkeyPatch, model: str, modify_params: bool, keeps_thinking: bool, keeps_edit: bool
) -> None:
monkeypatch.setattr(litellm, "modify_params", modify_params)
result = AnthropicConfig().transform_request(
result: Final = AnthropicConfig().transform_request(
model=model,
messages=[
{"role": "user", "content": "Look it up"},

24
tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py
View file

@ -368,8 +368,10 @@ CLEAR_TOOL_USES_EDIT: Final = {"type": "clear_tool_uses_20250919"}
),
],
)
def test_clear_thinking_edit_dropped_exactly_when_thinking_is(model, params, keeps_thinking):
result = _transform(
def test_clear_thinking_edit_dropped_exactly_when_thinking_is(
model: str, params: dict[str, object], keeps_thinking: bool
) -> None:
result: Final = _transform(
model,
{
"max_tokens": 8192,
@ -379,13 +381,23 @@ def test_clear_thinking_edit_dropped_exactly_when_thinking_is(model, params, kee
)
assert ("thinking" in result) is keeps_thinking
expected_edits = [CLEAR_THINKING_EDIT, CLEAR_TOOL_USES_EDIT] if keeps_thinking else [CLEAR_TOOL_USES_EDIT]
expected_edits: Final = [CLEAR_THINKING_EDIT, CLEAR_TOOL_USES_EDIT] if keeps_thinking else [CLEAR_TOOL_USES_EDIT]
assert result["context_management"] == {"edits": expected_edits}
@pytest.mark.parametrize("model", ["claude-fable-5", "claude-sonnet-5", "claude-opus-5", "claude-haiku-5-5"])
def test_clear_thinking_edit_kept_when_model_thinks_without_thinking_param(model):
result = _transform(
@pytest.mark.parametrize(
"model",
[
"claude-fable-5",
"claude-sonnet-5",
"claude-opus-5",
"claude-haiku-5-5",
pytest.param("claude-sonnet-5-1", id="unmapped-claude-5-minor"),
pytest.param("claude-opus-5-2-20270101", id="unmapped-dated-claude-5-minor"),
],
)
def test_clear_thinking_edit_kept_when_model_thinks_without_thinking_param(model: str) -> None:
result: Final = _transform(
model,
{"max_tokens": 8192, "reasoning_effort": "none", "context_management": {"edits": [CLEAR_THINKING_EDIT]}},
)

8
tests/unit/llms/anthropic/test_anthropic_common_utils.py
View file

@ -1611,12 +1611,12 @@ class TestAnthropicThinkingSignatureSelfHeal:
assert "thinking" not in data
assert data["messages"] == []
def test_strip_thinking_request_dict_drops_clear_thinking_edits(self):
def test_strip_thinking_request_dict_drops_clear_thinking_edits(self) -> None:
from litellm.llms.anthropic.common_utils import (
strip_thinking_blocks_from_anthropic_messages_request_dict,
)
data = {
data: Final[dict[str, object]] = {
"messages": [{"role": "assistant", "content": [{"type": "thinking", "thinking": "unsigned"}]}],
"thinking": {"type": "adaptive"},
"context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]},
@ -1648,12 +1648,12 @@ class TestAnthropicThinkingSignatureSelfHeal:
),
],
)
def test_drop_clear_thinking_edits(self, context_management, expected):
def test_drop_clear_thinking_edits(self, context_management: object, expected: object) -> None:
from litellm.llms.anthropic.common_utils import (
drop_clear_thinking_edits,
)
params = {"context_management": context_management}
params: Final[dict[str, object]] = {"context_management": context_management}
drop_clear_thinking_edits(params)
assert params == ({} if expected is None else {"context_management": expected})

11
tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py
View file

@ -2202,10 +2202,12 @@ def test_bedrock_invoke_transform_emits_adaptive_thinking_for_opus_4_8():
pytest.param({"reasoning_effort": "none"}, id="reasoning-effort-none"),
],
)
def test_bedrock_invoke_keeps_thinking_for_clear_thinking_edit_when_base_drops_thinking(thinking_params):
def test_bedrock_invoke_keeps_thinking_for_clear_thinking_edit_when_base_drops_thinking(
thinking_params: dict[str, object],
) -> None:
from litellm.types.router import GenericLiteLLMParams
result = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
model="anthropic.claude-opus-4-5-20251101-v1:0",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_optional_request_params={
@ -4516,7 +4518,10 @@ async def test_bedrock_messages_fallback_to_anthropic_sends_the_tool_names_bedro
"aws_region_name": "us-east-1",
},
},
{"model_name": "anthropic-claude", "litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "test"}},
{
"model_name": "anthropic-claude",
"litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "test"},
},
],
fallbacks=[{"bedrock-claude": ["anthropic-claude"]}],
num_retries=0,

21
tests/unit/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py
View file

@ -249,6 +249,13 @@ class TestRequestBody:
["clear_thinking_20251015", "clear_tool_uses_20250919"],
id="thinking-added-back-keeps-edit",
),
pytest.param(
"anthropic.claude-opus-4-5-20251101-v1:0",
{"max_tokens": BEDROCK_MIN_THINKING_BUDGET_TOKENS, "reasoning_effort": "none"},
None,
["clear_tool_uses_20250919"],
id="max-tokens-too-small-to-add-thinking-back-drops-edit",
),
pytest.param(
"anthropic.claude-mythos-preview",
{},
@ -266,9 +273,13 @@ class TestRequestBody:
],
)
def test_clear_thinking_edit_dropped_only_with_dropped_thinking(
self, model, thinking_params, expected_thinking, expected_edit_types
):
body = BedrockMantleAnthropicMessagesConfig().transform_anthropic_messages_request(
self,
model: str,
thinking_params: dict[str, object],
expected_thinking: dict[str, object] | None,
expected_edit_types: list[str],
) -> None:
body: Final = BedrockMantleAnthropicMessagesConfig().transform_anthropic_messages_request(
model=model,
messages=[{"role": "user", "content": "ping"}],
anthropic_messages_optional_request_params={
@ -284,8 +295,8 @@ class TestRequestBody:
assert body.get("thinking") == expected_thinking
assert [edit["type"] for edit in body["context_management"]["edits"]] == expected_edit_types
def test_disabled_thinking_without_edits_is_untouched(self):
body = BedrockMantleAnthropicMessagesConfig().transform_anthropic_messages_request(
def test_disabled_thinking_without_edits_is_untouched(self) -> None:
body: Final = BedrockMantleAnthropicMessagesConfig().transform_anthropic_messages_request(
model="anthropic.claude-3-5-haiku-20241022-v1:0",
messages=[{"role": "user", "content": "ping"}],
anthropic_messages_optional_request_params={"max_tokens": 8, "thinking": {"type": "disabled"}},