mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge pull request #20338 from Lucky-Lodhi2004/ttl-prompt-caching-bedrock
fix #20326 - [Feature]: Support TTL(1h) field in prompt caching for Bedrock Claude 4.5 models
This commit is contained in:
Sameer Kankute
GitHub
commit
34e5bb29a5
6 changed files with 191 additions and 94 deletions
|
|
@ -599,9 +599,9 @@ class OpenTelemetry(CustomLogger):
|
|||
|
||||
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
|
||||
"""Extract dynamic headers from kwargs if available."""
|
||||
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
|
||||
kwargs.get("standard_callback_dynamic_params")
|
||||
)
|
||||
standard_callback_dynamic_params: Optional[
|
||||
StandardCallbackDynamicParams
|
||||
] = kwargs.get("standard_callback_dynamic_params")
|
||||
|
||||
if not standard_callback_dynamic_params:
|
||||
return None
|
||||
|
|
@ -619,7 +619,9 @@ class OpenTelemetry(CustomLogger):
|
|||
# Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
|
||||
cache_key = str(sorted(dynamic_headers.items()))
|
||||
if cache_key in self._tracer_provider_cache:
|
||||
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
|
||||
return self._tracer_provider_cache[cache_key].get_tracer(
|
||||
LITELLM_TRACER_NAME
|
||||
)
|
||||
|
||||
# Create a temporary tracer provider with dynamic headers
|
||||
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
|
|
@ -1618,7 +1620,6 @@ class OpenTelemetry(CustomLogger):
|
|||
|
||||
for idx, choice in enumerate(response_obj.get("choices")):
|
||||
if choice.get("finish_reason"):
|
||||
|
||||
message = choice.get("message")
|
||||
tool_calls = message.get("tool_calls")
|
||||
if tool_calls:
|
||||
|
|
@ -1631,7 +1632,9 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
self.handle_callback_failure(callback_name=self.callback_name or "opentelemetry")
|
||||
self.handle_callback_failure(
|
||||
callback_name=self.callback_name or "opentelemetry"
|
||||
)
|
||||
verbose_logger.exception(
|
||||
"OpenTelemetry logging error in set_attributes %s", str(e)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.types.llms.bedrock import *
|
||||
|
||||
from ..common_utils import is_claude_4_5_on_bedrock
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
|
|
@ -306,9 +308,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
return "nova-2-lite" in model_without_region
|
||||
|
||||
def _map_web_search_options(
|
||||
self,
|
||||
web_search_options: dict,
|
||||
model: str
|
||||
self, web_search_options: dict, model: str
|
||||
) -> Optional[BedrockToolBlock]:
|
||||
"""
|
||||
Map web_search_options to Nova grounding systemTool.
|
||||
|
|
@ -634,7 +634,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
Filtered list of beta headers
|
||||
"""
|
||||
filtered_betas = []
|
||||
|
||||
|
||||
# 1. Filter out beta headers that are universally unsupported on Bedrock Converse
|
||||
for beta in beta_list:
|
||||
should_keep = True
|
||||
|
|
@ -642,10 +642,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if unsupported_pattern in beta.lower():
|
||||
should_keep = False
|
||||
break
|
||||
|
||||
|
||||
if should_keep:
|
||||
filtered_betas.append(beta)
|
||||
|
||||
|
||||
return filtered_betas
|
||||
|
||||
def _separate_computer_use_tools(
|
||||
|
|
@ -808,11 +808,11 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if param == "web_search_options" and isinstance(value, dict):
|
||||
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
||||
# because empty dict {} is falsy but is a valid way to enable Nova grounding
|
||||
grounding_tool = self._map_web_search_options(value, model)
|
||||
if grounding_tool is not None:
|
||||
optional_params = self._add_tools_to_optional_params(
|
||||
optional_params=optional_params, tools=[grounding_tool]
|
||||
)
|
||||
grounding_tool = self._map_web_search_options(value, model)
|
||||
if grounding_tool is not None:
|
||||
optional_params = self._add_tools_to_optional_params(
|
||||
optional_params=optional_params, tools=[grounding_tool]
|
||||
)
|
||||
|
||||
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
|
||||
# Nova Lite 2 handles token budgeting differently through reasoningConfig
|
||||
|
|
@ -926,6 +926,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["system"],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[SystemContentBlock]:
|
||||
pass
|
||||
|
||||
|
|
@ -939,6 +940,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["content_block"],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[ContentBlock]:
|
||||
pass
|
||||
|
||||
|
|
@ -951,16 +953,26 @@ class AmazonConverseConfig(BaseConfig):
|
|||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["system", "content_block"],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[Union[SystemContentBlock, ContentBlock]]:
|
||||
if message_block.get("cache_control", None) is None:
|
||||
cache_control = message_block.get("cache_control", None)
|
||||
if cache_control is None:
|
||||
return None
|
||||
|
||||
cache_point = CachePointBlock(type="default")
|
||||
if isinstance(cache_control, dict) and "ttl" in cache_control:
|
||||
ttl = cache_control["ttl"]
|
||||
if ttl in ["5m", "1h"] and model is not None:
|
||||
if is_claude_4_5_on_bedrock(model):
|
||||
cache_point["ttl"] = ttl
|
||||
|
||||
if block_type == "system":
|
||||
return SystemContentBlock(cachePoint=CachePointBlock(type="default"))
|
||||
return SystemContentBlock(cachePoint=cache_point)
|
||||
else:
|
||||
return ContentBlock(cachePoint=CachePointBlock(type="default"))
|
||||
return ContentBlock(cachePoint=cache_point)
|
||||
|
||||
def _transform_system_message(
|
||||
self, messages: List[AllMessageValues]
|
||||
self, messages: List[AllMessageValues], model: Optional[str] = None
|
||||
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
|
||||
system_prompt_indices = []
|
||||
system_content_blocks: List[SystemContentBlock] = []
|
||||
|
|
@ -972,7 +984,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
SystemContentBlock(text=message["content"])
|
||||
)
|
||||
cache_block = self._get_cache_point_block(
|
||||
message, block_type="system"
|
||||
message, block_type="system", model=model
|
||||
)
|
||||
if cache_block:
|
||||
system_content_blocks.append(cache_block)
|
||||
|
|
@ -983,7 +995,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
SystemContentBlock(text=m["text"])
|
||||
)
|
||||
cache_block = self._get_cache_point_block(
|
||||
m, block_type="system"
|
||||
m, block_type="system", model=model
|
||||
)
|
||||
if cache_block:
|
||||
system_content_blocks.append(cache_block)
|
||||
|
|
@ -1137,13 +1149,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if beta not in seen:
|
||||
unique_betas.append(beta)
|
||||
seen.add(beta)
|
||||
|
||||
|
||||
# Filter out unsupported beta headers for Bedrock Converse API
|
||||
filtered_betas = self._filter_unsupported_beta_headers_for_bedrock(
|
||||
model=model,
|
||||
beta_list=unique_betas,
|
||||
)
|
||||
|
||||
|
||||
additional_request_params["anthropic_beta"] = filtered_betas
|
||||
|
||||
return bedrock_tools, anthropic_beta_list
|
||||
|
|
@ -1196,9 +1208,11 @@ class AmazonConverseConfig(BaseConfig):
|
|||
)
|
||||
|
||||
# Prepare and separate parameters
|
||||
inference_params, additional_request_params, request_metadata = self._prepare_request_params(
|
||||
optional_params, model
|
||||
)
|
||||
(
|
||||
inference_params,
|
||||
additional_request_params,
|
||||
request_metadata,
|
||||
) = self._prepare_request_params(optional_params, model)
|
||||
|
||||
original_tools = inference_params.pop("tools", [])
|
||||
|
||||
|
|
@ -1250,7 +1264,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: Optional[dict] = None,
|
||||
) -> RequestObject:
|
||||
messages, system_content_blocks = self._transform_system_message(messages)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=model
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
messages = self._convert_consecutive_user_messages_to_guarded_text(
|
||||
|
|
@ -1306,7 +1322,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: Optional[dict] = None,
|
||||
) -> RequestObject:
|
||||
messages, system_content_blocks = self._transform_system_message(messages)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=model
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
messages = self._convert_consecutive_user_messages_to_guarded_text(
|
||||
|
|
@ -1484,7 +1502,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return message, returned_finish_reason
|
||||
|
||||
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
|
||||
def _translate_message_content(
|
||||
self, content_blocks: List[ContentBlock]
|
||||
) -> Tuple[
|
||||
str,
|
||||
List[ChatCompletionToolCallChunk],
|
||||
Optional[List[BedrockConverseReasoningContentBlock]],
|
||||
|
|
@ -1501,9 +1521,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
reasoningContentBlocks: Optional[
|
||||
List[BedrockConverseReasoningContentBlock]
|
||||
] = None
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
for idx, content in enumerate(content_blocks):
|
||||
"""
|
||||
|
|
@ -1557,7 +1577,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
|
||||
|
||||
def _transform_response( # noqa: PLR0915
|
||||
def _transform_response( # noqa: PLR0915
|
||||
self,
|
||||
model: str,
|
||||
response: httpx.Response,
|
||||
|
|
@ -1630,9 +1650,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
reasoningContentBlocks: Optional[
|
||||
List[BedrockConverseReasoningContentBlock]
|
||||
] = None
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
|
||||
if message is not None:
|
||||
|
|
@ -1651,15 +1671,17 @@ class AmazonConverseConfig(BaseConfig):
|
|||
provider_specific_fields["citationsContent"] = citationsContentBlocks
|
||||
|
||||
if provider_specific_fields:
|
||||
chat_completion_message["provider_specific_fields"] = provider_specific_fields
|
||||
chat_completion_message[
|
||||
"provider_specific_fields"
|
||||
] = provider_specific_fields
|
||||
|
||||
if reasoningContentBlocks is not None:
|
||||
chat_completion_message["reasoning_content"] = (
|
||||
self._transform_reasoning_content(reasoningContentBlocks)
|
||||
)
|
||||
chat_completion_message["thinking_blocks"] = (
|
||||
self._transform_thinking_blocks(reasoningContentBlocks)
|
||||
)
|
||||
chat_completion_message[
|
||||
"reasoning_content"
|
||||
] = self._transform_reasoning_content(reasoningContentBlocks)
|
||||
chat_completion_message[
|
||||
"thinking_blocks"
|
||||
] = self._transform_thinking_blocks(reasoningContentBlocks)
|
||||
chat_completion_message["content"] = content_str
|
||||
if (
|
||||
json_mode is True
|
||||
|
|
|
|||
|
|
@ -446,6 +446,29 @@ def get_bedrock_base_model(model: str) -> str:
|
|||
return model
|
||||
|
||||
|
||||
def is_claude_4_5_on_bedrock(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is a Claude 4.5 model on Bedrock.
|
||||
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
claude_4_5_patterns = [
|
||||
"sonnet-4.5",
|
||||
"sonnet_4.5",
|
||||
"sonnet-4-5",
|
||||
"sonnet_4_5",
|
||||
"haiku-4.5",
|
||||
"haiku_4.5",
|
||||
"haiku-4-5",
|
||||
"haiku_4_5",
|
||||
"opus-4.5",
|
||||
"opus_4.5",
|
||||
"opus-4-5",
|
||||
"opus_4_5",
|
||||
]
|
||||
return any(pattern in model_lower for pattern in claude_4_5_patterns)
|
||||
|
||||
|
||||
# Import after standalone functions to avoid circular imports
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
|
||||
|
|
@ -815,21 +838,23 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
|
|||
# If it's already a list, return it
|
||||
if isinstance(anthropic_beta_header, list):
|
||||
return anthropic_beta_header
|
||||
|
||||
|
||||
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
|
||||
if isinstance(anthropic_beta_header, str):
|
||||
anthropic_beta_header = anthropic_beta_header.strip()
|
||||
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
|
||||
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith(
|
||||
"]"
|
||||
):
|
||||
try:
|
||||
parsed = json.loads(anthropic_beta_header)
|
||||
if isinstance(parsed, list):
|
||||
return [str(beta).strip() for beta in parsed]
|
||||
except json.JSONDecodeError:
|
||||
pass # Fall through to comma-separated parsing
|
||||
|
||||
|
||||
# Fall back to comma-separated values
|
||||
return [beta.strip() for beta in anthropic_beta_header.split(",")]
|
||||
|
||||
|
||||
return []
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
get_anthropic_beta_from_headers,
|
||||
is_claude_4_5_on_bedrock,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -54,7 +57,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
# These will be filtered out to prevent 400 "invalid beta flag" errors
|
||||
UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [
|
||||
"advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers
|
||||
"prompt-caching-scope"
|
||||
"prompt-caching-scope",
|
||||
]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -116,15 +119,22 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
)
|
||||
|
||||
def _remove_ttl_from_cache_control(
|
||||
self, anthropic_messages_request: Dict
|
||||
self, anthropic_messages_request: Dict, model: Optional[str] = None
|
||||
) -> None:
|
||||
"""
|
||||
Remove `ttl` field from cache_control in messages.
|
||||
Bedrock doesn't support the ttl field in cache_control.
|
||||
|
||||
Update: Bedock supports `5m` and `1h` for Claude 4.5 models.
|
||||
|
||||
Args:
|
||||
anthropic_messages_request: The request dictionary to modify in-place
|
||||
model: The model name to check if it supports ttl
|
||||
"""
|
||||
is_claude_4_5 = False
|
||||
if model:
|
||||
is_claude_4_5 = self._is_claude_4_5_on_bedrock(model)
|
||||
|
||||
if "messages" in anthropic_messages_request:
|
||||
for message in anthropic_messages_request["messages"]:
|
||||
if isinstance(message, dict) and "content" in message:
|
||||
|
|
@ -133,7 +143,14 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
for item in content:
|
||||
if isinstance(item, dict) and "cache_control" in item:
|
||||
cache_control = item["cache_control"]
|
||||
if isinstance(cache_control, dict) and "ttl" in cache_control:
|
||||
if (
|
||||
isinstance(cache_control, dict)
|
||||
and "ttl" in cache_control
|
||||
):
|
||||
ttl = cache_control["ttl"]
|
||||
if is_claude_4_5 and ttl in ["5m", "1h"]:
|
||||
continue
|
||||
|
||||
cache_control.pop("ttl", None)
|
||||
|
||||
def _supports_extended_thinking_on_bedrock(self, model: str) -> bool:
|
||||
|
|
@ -155,10 +172,18 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
# Supported models on Bedrock for extended thinking
|
||||
supported_patterns = [
|
||||
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", # Opus 4.5
|
||||
"opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", # Opus 4.1
|
||||
"opus-4", "opus_4", # Opus 4
|
||||
"sonnet-4", "sonnet_4", # Sonnet 4
|
||||
"opus-4.5",
|
||||
"opus_4.5",
|
||||
"opus-4-5",
|
||||
"opus_4_5", # Opus 4.5
|
||||
"opus-4.1",
|
||||
"opus_4.1",
|
||||
"opus-4-1",
|
||||
"opus_4_1", # Opus 4.1
|
||||
"opus-4",
|
||||
"opus_4", # Opus 4
|
||||
"sonnet-4",
|
||||
"sonnet_4", # Sonnet 4
|
||||
]
|
||||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
|
@ -175,10 +200,27 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
"""
|
||||
model_lower = model.lower()
|
||||
opus_4_5_patterns = [
|
||||
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
|
||||
"opus-4.5",
|
||||
"opus_4.5",
|
||||
"opus-4-5",
|
||||
"opus_4_5",
|
||||
]
|
||||
return any(pattern in model_lower for pattern in opus_4_5_patterns)
|
||||
|
||||
def _is_claude_4_5_on_bedrock(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Claude 4.5 on Bedrock.
|
||||
|
||||
Claude Sonnet 4.5, Haiku 4.5, and Opus 4.5 support 1-hour prompt caching.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
True if the model is Claude 4.5
|
||||
"""
|
||||
return is_claude_4_5_on_bedrock(model)
|
||||
|
||||
def _supports_tool_search_on_bedrock(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model supports tool search on Bedrock.
|
||||
|
|
@ -199,9 +241,15 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
# Supported models for tool search on Bedrock
|
||||
supported_patterns = [
|
||||
# Opus 4.5
|
||||
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
|
||||
"opus-4.5",
|
||||
"opus_4.5",
|
||||
"opus-4-5",
|
||||
"opus_4_5",
|
||||
# Sonnet 4.5
|
||||
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
|
||||
"sonnet-4.5",
|
||||
"sonnet_4.5",
|
||||
"sonnet-4-5",
|
||||
"sonnet_4_5",
|
||||
]
|
||||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
|
@ -238,8 +286,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
beta_headers_to_remove.add(beta)
|
||||
has_advanced_tool_use = True
|
||||
break
|
||||
|
||||
|
||||
|
||||
# 2. Filter out extended thinking headers for models that don't support them
|
||||
extended_thinking_patterns = [
|
||||
"extended-thinking",
|
||||
|
|
@ -263,7 +310,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
beta_set.add("tool-examples-2025-10-29")
|
||||
|
||||
|
||||
def _get_tool_search_beta_header_for_bedrock(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -290,7 +336,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
input_examples_used: Whether input examples are used
|
||||
beta_set: The set of beta headers to modify in-place
|
||||
"""
|
||||
if tool_search_used and not (programmatic_tool_calling_used or input_examples_used):
|
||||
if tool_search_used and not (
|
||||
programmatic_tool_calling_used or input_examples_used
|
||||
):
|
||||
beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
|
||||
if "opus-4" in model.lower() or "opus_4" in model.lower():
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
|
|
@ -302,13 +350,13 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
) -> None:
|
||||
"""
|
||||
Convert Anthropic output_format to inline schema in message content.
|
||||
|
||||
|
||||
Bedrock Invoke doesn't support the output_format parameter, so we embed
|
||||
the schema directly into the user message content as text instructions.
|
||||
|
||||
|
||||
This approach adds the schema to the last user message, instructing the model
|
||||
to respond in the specified JSON format.
|
||||
|
||||
|
||||
Args:
|
||||
output_format: The output_format dict with 'type' and 'schema'
|
||||
anthropic_messages_request: The request dict to modify in-place
|
||||
|
|
@ -321,35 +369,32 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
schema = output_format.get("schema")
|
||||
if not schema:
|
||||
return
|
||||
|
||||
|
||||
# Get messages from the request
|
||||
messages = anthropic_messages_request.get("messages", [])
|
||||
if not messages:
|
||||
return
|
||||
|
||||
|
||||
# Find the last user message
|
||||
last_user_message_idx = None
|
||||
for idx in range(len(messages) - 1, -1, -1):
|
||||
if messages[idx].get("role") == "user":
|
||||
last_user_message_idx = idx
|
||||
break
|
||||
|
||||
|
||||
if last_user_message_idx is None:
|
||||
return
|
||||
|
||||
|
||||
last_user_message = messages[last_user_message_idx]
|
||||
content = last_user_message.get("content", [])
|
||||
|
||||
|
||||
# Ensure content is a list
|
||||
if isinstance(content, str):
|
||||
content = [{"type": "text", "text": content}]
|
||||
last_user_message["content"] = content
|
||||
|
||||
|
||||
# Add schema as text content to the message
|
||||
schema_text = {
|
||||
"type": "text",
|
||||
"text": json.dumps(schema)
|
||||
}
|
||||
schema_text = {"type": "text", "text": json.dumps(schema)}
|
||||
content.append(schema_text)
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
|
|
@ -374,9 +419,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
# 1. anthropic_version is required for all claude models
|
||||
if "anthropic_version" not in anthropic_messages_request:
|
||||
anthropic_messages_request["anthropic_version"] = (
|
||||
self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
|
||||
)
|
||||
anthropic_messages_request[
|
||||
"anthropic_version"
|
||||
] = self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
|
||||
|
||||
# 2. `stream` is not allowed in request body for bedrock invoke
|
||||
if "stream" in anthropic_messages_request:
|
||||
|
|
@ -386,8 +431,10 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if "model" in anthropic_messages_request:
|
||||
anthropic_messages_request.pop("model", None)
|
||||
|
||||
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it)
|
||||
self._remove_ttl_from_cache_control(anthropic_messages_request)
|
||||
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models)
|
||||
self._remove_ttl_from_cache_control(
|
||||
anthropic_messages_request=anthropic_messages_request, model=model
|
||||
)
|
||||
|
||||
# 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format)
|
||||
output_format = anthropic_messages_request.pop("output_format", None)
|
||||
|
|
@ -396,14 +443,14 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
output_format=output_format,
|
||||
anthropic_messages_request=anthropic_messages_request,
|
||||
)
|
||||
|
||||
|
||||
# 6. AUTO-INJECT beta headers based on features used
|
||||
anthropic_model_info = AnthropicModelInfo()
|
||||
tools = anthropic_messages_optional_request_params.get("tools")
|
||||
messages_typed = cast(List[AllMessageValues], messages)
|
||||
tool_search_used = anthropic_model_info.is_tool_search_used(tools)
|
||||
programmatic_tool_calling_used = anthropic_model_info.is_programmatic_tool_calling_used(
|
||||
tools
|
||||
programmatic_tool_calling_used = (
|
||||
anthropic_model_info.is_programmatic_tool_calling_used(tools)
|
||||
)
|
||||
input_examples_used = anthropic_model_info.is_input_examples_used(tools)
|
||||
|
||||
|
|
@ -436,8 +483,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
if beta_set:
|
||||
anthropic_messages_request["anthropic_beta"] = list(beta_set)
|
||||
|
||||
|
||||
|
||||
return anthropic_messages_request
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
|
|
@ -455,7 +501,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
)
|
||||
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
|
||||
return self.bedrock_sse_wrapper(
|
||||
completion_stream=completion_stream,
|
||||
completion_stream=completion_stream,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
request_body=request_body,
|
||||
)
|
||||
|
|
@ -474,14 +520,14 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
)
|
||||
|
||||
handler = BaseAnthropicMessagesStreamingIterator(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
request_body=request_body,
|
||||
)
|
||||
|
||||
|
||||
async for chunk in handler.async_sse_wrapper(completion_stream):
|
||||
yield chunk
|
||||
|
||||
|
||||
|
||||
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import List, Optional, Tuple
|
||||
|
||||
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -29,9 +30,7 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
api_key: Optional[str],
|
||||
custom_llm_provider: str,
|
||||
) -> Tuple[Optional[str], Optional[str], str]:
|
||||
dynamic_api_base = (
|
||||
self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
dynamic_api_base = self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
|
||||
try:
|
||||
dynamic_api_key = self.authenticator.get_api_key()
|
||||
except GetAPIKeyError as e:
|
||||
|
|
@ -140,7 +139,7 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
"""
|
||||
Check if any message contains vision content (images).
|
||||
Returns True if any message has content with vision-related types, otherwise False.
|
||||
|
||||
|
||||
Checks for:
|
||||
- image_url content type (OpenAI format)
|
||||
- Content items with type 'image_url'
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from .openai import ChatCompletionToolCallChunk
|
|||
|
||||
class CachePointBlock(TypedDict, total=False):
|
||||
type: Literal["default"]
|
||||
ttl: str
|
||||
|
||||
|
||||
class SystemContentBlock(TypedDict, total=False):
|
||||
|
|
@ -961,6 +962,7 @@ class BedrockGetBatchResponse(TypedDict, total=False):
|
|||
timeoutDurationInHours: Optional[int]
|
||||
clientRequestToken: Optional[str]
|
||||
|
||||
|
||||
class BedrockToolBlock(TypedDict, total=False):
|
||||
toolSpec: Optional[ToolSpecBlock]
|
||||
systemTool: Optional[SystemToolBlock] # For Nova grounding
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue