mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): keep thinking on the signature retry when a clear_thinking edit needs it
The invalid-signature retry strips the signed thinking blocks and used to drop thinking too, which left a clear_thinking edit Anthropic rejects. It now keeps thinking when the request carries that edit, so the retried turn still thinks, and drops it as before otherwise. Dropping a clear_thinking edit elsewhere now logs a warning that names model_info.thinking_on_by_default for deployments the cost map cannot resolve.
This commit is contained in:
parent
4937b1e80b
commit
7fb6207676
2 changed files with 85 additions and 17 deletions
|
|
@ -2020,30 +2020,32 @@ def _is_clear_thinking_edit(edit: object) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def drop_clear_thinking_edits(params: MutableMapping[str, object]) -> None: # mutable-ok: edits in place
|
||||
def drop_clear_thinking_edits(params: MutableMapping[str, object]) -> bool: # mutable-ok: edits in place
|
||||
"""
|
||||
Remove the ``clear_thinking_*`` context-management edits, which Anthropic
|
||||
rejects without ``thinking``. ``context_management`` itself goes only when
|
||||
such an edit was removed and nothing else is left in it.
|
||||
such an edit was removed and nothing else is left in it. Returns whether an
|
||||
edit was removed.
|
||||
"""
|
||||
raw_context_management: Final = params.get("context_management")
|
||||
if not isinstance(raw_context_management, dict):
|
||||
return
|
||||
return False
|
||||
context_management: Final = OBJECT_MAPPING_ADAPTER.validate_python(raw_context_management)
|
||||
raw_edits: Final = context_management.get("edits")
|
||||
if not isinstance(raw_edits, list):
|
||||
return
|
||||
return False
|
||||
edits: Final = OBJECT_LIST_ADAPTER.validate_python(raw_edits)
|
||||
kept_edits: Final = [edit for edit in edits if not _is_clear_thinking_edit(edit)]
|
||||
if len(kept_edits) == len(edits):
|
||||
return
|
||||
return False
|
||||
if not kept_edits and len(context_management) == 1:
|
||||
params.pop("context_management")
|
||||
return
|
||||
return True
|
||||
params["context_management"] = { # rebind-ok: callers mutate their request params in place
|
||||
**context_management,
|
||||
"edits": kept_edits,
|
||||
}
|
||||
return True
|
||||
|
||||
|
||||
def drop_clear_thinking_edits_for_model(
|
||||
|
|
@ -2056,8 +2058,23 @@ def drop_clear_thinking_edits_for_model(
|
|||
AnthropicModelInfo.supports_model_capability(model, capability, custom_llm_provider)
|
||||
for capability in ("thinking_always_on", "thinking_on_by_default")
|
||||
)
|
||||
if not thinks_without_thinking_param:
|
||||
drop_clear_thinking_edits(params)
|
||||
if not thinks_without_thinking_param and drop_clear_thinking_edits(params):
|
||||
litellm.verbose_logger.warning(
|
||||
"litellm removed the clear_thinking context-management edit for model %s because the request goes out "
|
||||
"without thinking. If this model thinks by default, set model_info.thinking_on_by_default to true to "
|
||||
"keep the edit.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def _has_clear_thinking_edit(params: Mapping[str, object]) -> bool:
|
||||
raw_context_management: Final = params.get("context_management")
|
||||
if not isinstance(raw_context_management, dict):
|
||||
return False
|
||||
raw_edits: Final = OBJECT_MAPPING_ADAPTER.validate_python(raw_context_management).get("edits")
|
||||
if not isinstance(raw_edits, list):
|
||||
return False
|
||||
return any(_is_clear_thinking_edit(edit) for edit in OBJECT_LIST_ADAPTER.validate_python(raw_edits))
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
||||
|
|
@ -2065,14 +2082,13 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
|||
) -> None:
|
||||
"""
|
||||
Mutate an Anthropic Messages-style request dict: strip thinking blocks from
|
||||
``messages`` and remove ``thinking`` with its dependent context-management
|
||||
edits (see :func:`drop_clear_thinking_edits`).
|
||||
``messages`` and remove ``thinking`` unless a ``clear_thinking_*`` edit needs it.
|
||||
"""
|
||||
msgs: Final = data.get("messages")
|
||||
if isinstance(msgs, list):
|
||||
data["messages"] = strip_thinking_blocks_from_anthropic_messages(msgs)
|
||||
data.pop("thinking", None)
|
||||
drop_clear_thinking_edits(data)
|
||||
if not _has_clear_thinking_edit(data):
|
||||
data.pop("thinking", None)
|
||||
|
||||
|
||||
_DROP_ANTHROPIC_MESSAGE: Final = object()
|
||||
|
|
|
|||
|
|
@ -1704,18 +1704,47 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
assert "thinking" not in data
|
||||
assert data["messages"] == []
|
||||
|
||||
def test_strip_thinking_request_dict_drops_clear_thinking_edits(self) -> None:
|
||||
@pytest.mark.parametrize("thinking", [{"type": "adaptive"}, {"type": "enabled", "budget_tokens": 1024}])
|
||||
def test_strip_thinking_request_dict_keeps_thinking_for_a_clear_thinking_edit(
|
||||
self, thinking: dict[str, object]
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict,
|
||||
)
|
||||
|
||||
context_management: Final = {
|
||||
"edits": [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}]
|
||||
}
|
||||
data: Final[dict[str, object]] = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": [{"type": "thinking", "thinking": "unsigned"}]},
|
||||
],
|
||||
"thinking": thinking,
|
||||
"context_management": context_management,
|
||||
}
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict(data)
|
||||
assert data == {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": thinking,
|
||||
"context_management": context_management,
|
||||
}
|
||||
|
||||
def test_strip_thinking_request_dict_drops_thinking_without_a_clear_thinking_edit(self) -> None:
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict,
|
||||
)
|
||||
|
||||
data: Final[dict[str, object]] = {
|
||||
"messages": [{"role": "assistant", "content": [{"type": "thinking", "thinking": "unsigned"}]}],
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": {"type": "adaptive"},
|
||||
"context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]},
|
||||
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
}
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict(data)
|
||||
assert "context_management" not in data
|
||||
assert data == {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("context_management", "expected"),
|
||||
|
|
@ -1747,8 +1776,31 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
)
|
||||
|
||||
params: Final[dict[str, object]] = {"context_management": context_management}
|
||||
drop_clear_thinking_edits(params)
|
||||
dropped: Final = drop_clear_thinking_edits(params)
|
||||
assert params == ({} if expected is None else {"context_management": expected})
|
||||
assert dropped is (expected != context_management)
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "edits", "warned"),
|
||||
[
|
||||
pytest.param("claude-sonnet-4-5", [{"type": "clear_thinking_20251015"}], True, id="dropped-warns"),
|
||||
pytest.param("claude-sonnet-5-5", [{"type": "clear_thinking_20251015"}], False, id="kept-quiet"),
|
||||
pytest.param("claude-sonnet-4-5", [{"type": "clear_tool_uses_20250919"}], False, id="nothing-dropped"),
|
||||
],
|
||||
)
|
||||
def test_drop_clear_thinking_edits_for_model_warns_only_when_an_edit_goes(
|
||||
self, model: str, edits: list[dict[str, str]], warned: bool, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import drop_clear_thinking_edits_for_model
|
||||
|
||||
params: Final[dict[str, object]] = {"context_management": {"edits": edits}}
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
drop_clear_thinking_edits_for_model(params, model, "anthropic")
|
||||
warnings: Final = [r.getMessage() for r in caplog.records if "clear_thinking" in r.getMessage()]
|
||||
|
||||
assert len(warnings) == int(warned)
|
||||
assert all(model in w and "model_info.thinking_on_by_default" in w for w in warnings)
|
||||
|
||||
def test_strip_empty_content_blocks_from_anthropic_messages(self):
|
||||
"""Covers #22930. The core regression scenario: an assistant message
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue