mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(bedrock): inject thinking for clear_thinking context_management on Messages API
Bedrock rejects clear_thinking_20251015 unless thinking is enabled or adaptive. Inject minimal extended thinking and interleaved-thinking beta when Claude Code sends context_management without thinking. Adds unit tests. Made-with: Cursor
This commit is contained in:
parent
fb33daa09f
commit
607412defb
2 changed files with 129 additions and 0 deletions
|
|
@ -13,6 +13,8 @@ from typing import (
|
|||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.constants import BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
from litellm.litellm_core_utils.litellm_logging import verbose_logger
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -213,6 +215,70 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
||||
def _ensure_thinking_for_clear_thinking_context_management(
|
||||
self,
|
||||
anthropic_messages_request: Dict,
|
||||
model: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Bedrock rejects ``clear_thinking_20251015`` context-management edits unless
|
||||
extended thinking is ``enabled`` or ``adaptive``. Claude Code often sends
|
||||
context management without a top-level ``thinking`` field.
|
||||
|
||||
When we detect that edit type on a model that supports extended thinking on
|
||||
Bedrock, inject a minimal ``thinking`` config so the request succeeds.
|
||||
|
||||
Returns:
|
||||
True if ``thinking`` was added or upgraded for this fix (caller may
|
||||
need to add the interleaved-thinking beta header).
|
||||
"""
|
||||
cm = anthropic_messages_request.get("context_management")
|
||||
if not isinstance(cm, dict):
|
||||
return False
|
||||
edits = cm.get("edits")
|
||||
if not isinstance(edits, list):
|
||||
return False
|
||||
needs_thinking = any(
|
||||
isinstance(e, dict) and e.get("type") == "clear_thinking_20251015"
|
||||
for e in edits
|
||||
)
|
||||
if not needs_thinking:
|
||||
return False
|
||||
if not self._supports_extended_thinking_on_bedrock(model):
|
||||
return False
|
||||
|
||||
thinking = anthropic_messages_request.get("thinking")
|
||||
if isinstance(thinking, dict):
|
||||
t = thinking.get("type")
|
||||
if t in ("enabled", "adaptive"):
|
||||
return False
|
||||
# ``disabled`` or unknown — replace with enabled so clear_thinking is valid
|
||||
verbose_logger.debug(
|
||||
"Bedrock clear_thinking_20251015: replacing thinking=%s with minimal enabled thinking",
|
||||
thinking,
|
||||
)
|
||||
|
||||
max_tokens = anthropic_messages_request.get("max_tokens")
|
||||
budget = BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
if isinstance(max_tokens, int) and max_tokens <= budget:
|
||||
verbose_logger.warning(
|
||||
"Bedrock clear_thinking_20251015: max_tokens=%s is not greater than "
|
||||
"minimum thinking budget (%s); cannot inject thinking safely",
|
||||
max_tokens,
|
||||
budget,
|
||||
)
|
||||
return False
|
||||
|
||||
anthropic_messages_request["thinking"] = {
|
||||
"type": "enabled",
|
||||
"budget_tokens": budget,
|
||||
}
|
||||
verbose_logger.debug(
|
||||
"Bedrock clear_thinking_20251015: injected thinking with budget_tokens=%s",
|
||||
budget,
|
||||
)
|
||||
return True
|
||||
|
||||
def _is_claude_opus_4_5(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Claude Opus 4.5.
|
||||
|
|
@ -414,6 +480,13 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if "model" in anthropic_messages_request:
|
||||
anthropic_messages_request.pop("model", None)
|
||||
|
||||
injected_thinking_for_clear_thinking = (
|
||||
self._ensure_thinking_for_clear_thinking_context_management(
|
||||
anthropic_messages_request=anthropic_messages_request,
|
||||
model=model,
|
||||
)
|
||||
)
|
||||
|
||||
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models)
|
||||
self._remove_ttl_from_cache_control(
|
||||
anthropic_messages_request=anthropic_messages_request, model=model
|
||||
|
|
@ -463,6 +536,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
)
|
||||
beta_set.update(auto_betas)
|
||||
|
||||
if injected_thinking_for_clear_thinking:
|
||||
beta_set.add("interleaved-thinking-2025-05-14")
|
||||
|
||||
self._get_tool_search_beta_header_for_bedrock(
|
||||
model=model,
|
||||
tool_search_used=tool_search_used,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
remove_custom_field_from_tools,
|
||||
)
|
||||
from litellm.constants import BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
AmazonAnthropicClaudeMessagesStreamDecoder,
|
||||
|
|
@ -384,6 +385,58 @@ def test_bedrock_invoke_messages_transform_adds_name_when_tool_missing_name():
|
|||
assert result["tools"][0]["name"] == "litellm_unnamed_tool_0"
|
||||
|
||||
|
||||
def test_bedrock_invoke_messages_injects_thinking_for_clear_thinking_context_management():
|
||||
"""
|
||||
Bedrock requires extended thinking when ``clear_thinking_20251015`` appears in
|
||||
``context_management`` (Claude Code sends CM without ``thinking``).
|
||||
"""
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
optional_params = {
|
||||
"max_tokens": 32000,
|
||||
"stream": False,
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]
|
||||
},
|
||||
}
|
||||
result = cfg.transform_anthropic_messages_request(
|
||||
model="global.anthropic.claude-sonnet-4-6-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
anthropic_messages_optional_request_params=copy.deepcopy(optional_params),
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert result["thinking"]["type"] == "enabled"
|
||||
assert result["thinking"]["budget_tokens"] == BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
betas = result.get("anthropic_beta") or []
|
||||
assert "interleaved-thinking-2025-05-14" in betas
|
||||
|
||||
|
||||
def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled():
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
optional_params = {
|
||||
"max_tokens": 32000,
|
||||
"stream": False,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048},
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]
|
||||
},
|
||||
}
|
||||
result = cfg.transform_anthropic_messages_request(
|
||||
model="global.anthropic.claude-sonnet-4-6-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
anthropic_messages_optional_request_params=copy.deepcopy(optional_params),
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert result["thinking"]["budget_tokens"] == 2048
|
||||
betas = result.get("anthropic_beta") or []
|
||||
assert "interleaved-thinking-2025-05-14" not in betas
|
||||
|
||||
|
||||
def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_object():
|
||||
"""
|
||||
End-to-end: AmazonAnthropicClaudeMessagesConfig must emit Bedrock Invoke bodies
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue