mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
Messages api bedrock converse caching and pdf support (#19785)
* cache control for user messages and system messages * add cache createion tokens in reponse * cache controls in tool calls and assistant turns * refactor with _should_preserve_cache_control * add cache control unit tests * use simpler cache creation token count logic * use helper function * remove unused function * fix unit tests
This commit is contained in:
colinlin-stripe
GitHub
parent
f2c3a01a57
commit
7605062e94
2 changed files with 425 additions and 44 deletions
|
|
@ -168,6 +168,24 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return provider_specific_fields.get("signature")
|
||||
return None
|
||||
|
||||
def _add_cache_control_if_applicable(
|
||||
self,
|
||||
source: Dict[str, Any],
|
||||
target: Dict[str, Any],
|
||||
model: Optional[str],
|
||||
) -> None:
|
||||
"""
|
||||
Extract cache_control from source and add to target if it should be preserved.
|
||||
|
||||
Args:
|
||||
source: Dict containing potential cache_control field
|
||||
target: Dict to add cache_control to
|
||||
model: Model name to check if cache_control should be preserved
|
||||
"""
|
||||
cache_control = source.get("cache_control")
|
||||
if cache_control and model and self.is_anthropic_claude_model(model):
|
||||
target["cache_control"] = cache_control
|
||||
|
||||
def translatable_anthropic_params(self) -> List:
|
||||
"""
|
||||
Which anthropic params, we need to translate to the openai format.
|
||||
|
|
@ -202,15 +220,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
elif message_content and isinstance(message_content, list):
|
||||
for content in message_content:
|
||||
if content.get("type") == "text":
|
||||
text_obj = ChatCompletionTextObject(
|
||||
text_obj: Dict[str, Any] = ChatCompletionTextObject(
|
||||
type="text", text=content.get("text", "")
|
||||
)
|
||||
# Preserve cache_control if present (for prompt caching)
|
||||
# Only for Anthropic models that support prompt caching
|
||||
cache_control = content.get("cache_control")
|
||||
if cache_control and model and self.is_anthropic_claude_model(model):
|
||||
text_obj["cache_control"] = cache_control # type: ignore
|
||||
new_user_content_list.append(text_obj)
|
||||
self._add_cache_control_if_applicable(content, text_obj, model)
|
||||
new_user_content_list.append(text_obj) # type: ignore
|
||||
elif content.get("type") == "image":
|
||||
# Convert Anthropic image format to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -222,25 +236,44 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
image_url_obj = ChatCompletionImageUrlObject(
|
||||
url=openai_image_url
|
||||
)
|
||||
image_obj = ChatCompletionImageObject(
|
||||
image_obj: Dict[str, Any] = ChatCompletionImageObject(
|
||||
type="image_url", image_url=image_url_obj
|
||||
)
|
||||
new_user_content_list.append(image_obj)
|
||||
self._add_cache_control_if_applicable(content, image_obj, model)
|
||||
new_user_content_list.append(image_obj) # type: ignore
|
||||
elif content.get("type") == "document":
|
||||
# Convert Anthropic document format (PDF, etc.) to OpenAI format
|
||||
source = content.get("source", {})
|
||||
openai_image_url = (
|
||||
self._translate_anthropic_image_to_openai(source)
|
||||
)
|
||||
|
||||
if openai_image_url:
|
||||
image_url_obj = ChatCompletionImageUrlObject(
|
||||
url=openai_image_url
|
||||
)
|
||||
doc_obj: Dict[str, Any] = ChatCompletionImageObject(
|
||||
type="image_url", image_url=image_url_obj
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, doc_obj, model)
|
||||
new_user_content_list.append(doc_obj) # type: ignore
|
||||
elif content.get("type") == "tool_result":
|
||||
if "content" not in content:
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
tool_result: Dict[str, Any] = ChatCompletionToolMessage(
|
||||
role="tool",
|
||||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content="",
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
elif isinstance(content.get("content"), str):
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
role="tool",
|
||||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=str(content.get("content", "")),
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
elif isinstance(content.get("content"), list):
|
||||
# Combine all content items into a single tool message
|
||||
# to avoid creating multiple tool_result blocks with the same ID
|
||||
|
|
@ -256,7 +289,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=c,
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
elif isinstance(c, dict):
|
||||
if c.get("type") == "text":
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
|
|
@ -266,7 +300,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
),
|
||||
content=c.get("text", ""),
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
elif c.get("type") == "image":
|
||||
source = c.get("source", {})
|
||||
openai_image_url = (
|
||||
|
|
@ -282,7 +317,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
),
|
||||
content=openai_image_url,
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
else:
|
||||
# For multiple content items, combine into a single tool message
|
||||
# with list content to preserve all items while having one tool_use_id
|
||||
|
|
@ -331,7 +367,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=combined_content_parts, # type: ignore
|
||||
)
|
||||
tool_message_list.append(tool_result)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
|
||||
if len(tool_message_list) > 0:
|
||||
new_messages.extend(tool_message_list)
|
||||
|
|
@ -344,7 +381,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
## ASSISTANT MESSAGE ##
|
||||
assistant_message_str: Optional[str] = None
|
||||
tool_calls: List[ChatCompletionAssistantToolCall] = []
|
||||
assistant_content_list: List[Dict[str, Any]] = [] # For content blocks with cache_control
|
||||
has_cache_control_in_text = False
|
||||
tool_calls: List[Dict[str, Any]] = []
|
||||
thinking_blocks: List[
|
||||
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
||||
] = []
|
||||
|
|
@ -357,10 +396,14 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
assistant_message_str = str(content)
|
||||
elif isinstance(content, dict):
|
||||
if content.get("type") == "text":
|
||||
if assistant_message_str is None:
|
||||
assistant_message_str = content.get("text", "")
|
||||
else:
|
||||
assistant_message_str += content.get("text", "")
|
||||
text_block: Dict[str, Any] = {
|
||||
"type": "text",
|
||||
"text": content.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(content, text_block, model)
|
||||
if "cache_control" in text_block:
|
||||
has_cache_control_in_text = True
|
||||
assistant_content_list.append(text_block)
|
||||
elif content.get("type") == "tool_use":
|
||||
function_chunk: ChatCompletionToolCallFunctionChunk = {
|
||||
"name": content.get("name", ""),
|
||||
|
|
@ -384,13 +427,13 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
provider_specific_fields
|
||||
)
|
||||
|
||||
tool_calls.append(
|
||||
ChatCompletionAssistantToolCall(
|
||||
id=content.get("id", ""),
|
||||
type="function",
|
||||
function=function_chunk,
|
||||
)
|
||||
tool_call: Dict[str, Any] = ChatCompletionAssistantToolCall(
|
||||
id=content.get("id", ""),
|
||||
type="function",
|
||||
function=function_chunk,
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_call, model)
|
||||
tool_calls.append(tool_call)
|
||||
elif content.get("type") == "thinking":
|
||||
thinking_block = ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
|
|
@ -411,18 +454,30 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
if (
|
||||
assistant_message_str is not None
|
||||
or len(assistant_content_list) > 0
|
||||
or len(tool_calls) > 0
|
||||
or len(thinking_blocks) > 0
|
||||
):
|
||||
# Use list format if any text block has cache_control, otherwise use string
|
||||
if has_cache_control_in_text and len(assistant_content_list) > 0:
|
||||
assistant_content: Any = assistant_content_list
|
||||
elif len(assistant_content_list) > 0 and not has_cache_control_in_text:
|
||||
# Concatenate text blocks into string when no cache_control
|
||||
assistant_content = "".join(
|
||||
block.get("text", "") for block in assistant_content_list
|
||||
)
|
||||
else:
|
||||
assistant_content = assistant_message_str
|
||||
|
||||
assistant_message = ChatCompletionAssistantMessage(
|
||||
role="assistant",
|
||||
content=assistant_message_str,
|
||||
content=assistant_content,
|
||||
thinking_blocks=(
|
||||
thinking_blocks if len(thinking_blocks) > 0 else None
|
||||
),
|
||||
)
|
||||
if len(tool_calls) > 0:
|
||||
assistant_message["tool_calls"] = tool_calls
|
||||
assistant_message["tool_calls"] = tool_calls # type: ignore
|
||||
if len(thinking_blocks) > 0:
|
||||
assistant_message["thinking_blocks"] = thinking_blocks # type: ignore
|
||||
new_messages.append(assistant_message)
|
||||
|
|
@ -532,10 +587,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
def translate_anthropic_tools_to_openai(
|
||||
self, tools: List[AllAnthropicToolsValues]
|
||||
self, tools: List[AllAnthropicToolsValues], model: Optional[str] = None
|
||||
) -> List[ChatCompletionToolParam]:
|
||||
new_tools: List[ChatCompletionToolParam] = []
|
||||
mapped_tool_params = ["name", "input_schema", "description"]
|
||||
mapped_tool_params = ["name", "input_schema", "description", "cache_control"]
|
||||
for tool in tools:
|
||||
function_chunk = ChatCompletionToolParamFunctionChunk(
|
||||
name=tool["name"],
|
||||
|
|
@ -548,11 +603,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for k, v in tool.items():
|
||||
if k not in mapped_tool_params: # pass additional computer kwargs
|
||||
function_chunk.setdefault("parameters", {}).update({k: v})
|
||||
new_tools.append(
|
||||
ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
)
|
||||
tool_param: Dict[str, Any] = ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
self._add_cache_control_if_applicable(tool, tool_param, model)
|
||||
new_tools.append(tool_param) # type: ignore[arg-type]
|
||||
|
||||
return new_tools
|
||||
return new_tools # type: ignore[return-value]
|
||||
|
||||
def translate_anthropic_output_format_to_openai(
|
||||
self, output_format: Any
|
||||
|
|
@ -621,10 +676,29 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if "system" in anthropic_message_request:
|
||||
system_content = anthropic_message_request["system"]
|
||||
if system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=system_content),
|
||||
)
|
||||
# Handle system as string or array of content blocks
|
||||
if isinstance(system_content, str):
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=system_content),
|
||||
)
|
||||
elif isinstance(system_content, list):
|
||||
# Convert Anthropic system content blocks to OpenAI format
|
||||
openai_system_content: List[Dict[str, Any]] = []
|
||||
model_name = anthropic_message_request.get("model", "")
|
||||
for block in system_content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text_block: Dict[str, Any] = {
|
||||
"type": "text",
|
||||
"text": block.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(block, text_block, model_name)
|
||||
openai_system_content.append(text_block)
|
||||
if openai_system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore
|
||||
)
|
||||
|
||||
new_kwargs: ChatCompletionRequest = {
|
||||
"model": anthropic_message_request["model"],
|
||||
|
|
@ -655,7 +729,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tools = anthropic_message_request["tools"]
|
||||
if tools:
|
||||
new_kwargs["tools"] = self.translate_anthropic_tools_to_openai(
|
||||
tools=cast(List[AllAnthropicToolsValues], tools)
|
||||
tools=cast(List[AllAnthropicToolsValues], tools),
|
||||
model=new_kwargs.get("model"),
|
||||
)
|
||||
|
||||
## CONVERT THINKING
|
||||
|
|
@ -827,7 +902,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
# extract usage
|
||||
usage: Usage = getattr(response, "usage")
|
||||
anthropic_usage = AnthropicUsage(
|
||||
anthropic_usage: Dict[str, Any] = AnthropicUsage(
|
||||
input_tokens=usage.prompt_tokens or 0,
|
||||
output_tokens=usage.completion_tokens or 0,
|
||||
)
|
||||
|
|
@ -843,7 +918,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
role="assistant",
|
||||
model=response.model or "unknown-model",
|
||||
stop_sequence=None,
|
||||
usage=anthropic_usage,
|
||||
usage=anthropic_usage, # type: ignore
|
||||
content=anthropic_content, # type: ignore
|
||||
stop_reason=anthropic_finish_reason,
|
||||
)
|
||||
|
|
@ -980,7 +1055,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
else:
|
||||
litellm_usage_chunk = None
|
||||
if litellm_usage_chunk is not None:
|
||||
usage_delta = UsageDelta(
|
||||
usage_delta: Dict[str, Any] = UsageDelta(
|
||||
input_tokens=litellm_usage_chunk.prompt_tokens or 0,
|
||||
output_tokens=litellm_usage_chunk.completion_tokens or 0,
|
||||
)
|
||||
|
|
@ -992,7 +1067,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
else:
|
||||
usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
|
||||
return MessageBlockDelta(
|
||||
type="message_delta", delta=delta, usage=usage_delta
|
||||
type="message_delta", delta=delta, usage=usage_delta # type: ignore
|
||||
)
|
||||
(
|
||||
type_of_content,
|
||||
|
|
|
|||
|
|
@ -1108,3 +1108,309 @@ def test_streaming_chunk_with_both_text_and_tool_calls_issue_18238():
|
|||
assert block_type == "tool_use"
|
||||
assert content_block_start["name"] == "Bash"
|
||||
assert content_block_start["id"] == "toolu_bdrk_013xRVejhv3ybmLEGCoZib2b"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Cache Control Transformation Tests
|
||||
# ============================================================================
|
||||
|
||||
# Model constant for cache control tests
|
||||
CACHE_CONTROL_BEDROCK_CONVERSE_MODEL = "bedrock/converse/global.anthropic.claude-opus-4-5-20251101-v1:0"
|
||||
CACHE_CONTROL_NON_ANTHROPIC_MODEL = "gpt-4"
|
||||
|
||||
|
||||
def test_should_add_cache_control_for_anthropic_model():
|
||||
"""Should add cache_control to target for Anthropic Claude models."""
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
cache_control = {"type": "ephemeral"}
|
||||
|
||||
for model in [
|
||||
CACHE_CONTROL_BEDROCK_CONVERSE_MODEL,
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"claude-opus-4-5-20251101",
|
||||
"vertex_ai/claude-3-sonnet@20240229",
|
||||
]:
|
||||
target = {}
|
||||
adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model)
|
||||
assert "cache_control" in target
|
||||
assert target["cache_control"] == cache_control
|
||||
|
||||
|
||||
def test_should_not_add_cache_control_for_non_anthropic_model():
|
||||
"""Should not add cache_control for non-Anthropic models."""
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
cache_control = {"type": "ephemeral"}
|
||||
|
||||
for model in [CACHE_CONTROL_NON_ANTHROPIC_MODEL, "openai/gpt-4-turbo", "gemini-pro"]:
|
||||
target = {}
|
||||
adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model)
|
||||
assert "cache_control" not in target
|
||||
|
||||
|
||||
def test_should_not_add_cache_control_when_none():
|
||||
"""Should not add cache_control when source has None or empty cache_control."""
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
|
||||
for source in [{"cache_control": None}, {"cache_control": {}}, {"cache_control": ""}, {}]:
|
||||
target = {}
|
||||
adapter._add_cache_control_if_applicable(source, target, CACHE_CONTROL_BEDROCK_CONVERSE_MODEL)
|
||||
assert "cache_control" not in target
|
||||
|
||||
|
||||
def test_should_not_add_cache_control_when_model_none():
|
||||
"""Should not add cache_control when model is None or empty."""
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
cache_control = {"type": "ephemeral"}
|
||||
|
||||
for model in [None, ""]:
|
||||
target = {}
|
||||
adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model)
|
||||
assert "cache_control" not in target
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_text_content_for_claude():
|
||||
"""Cache control should be preserved in text content for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "This is cached content",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_not_preserved_for_non_claude_model():
|
||||
"""Cache control should NOT be preserved for non-Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "This is cached content",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "cache_control" not in result[0]["content"][0]
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_image_content_for_claude():
|
||||
"""Cache control should be preserved in image content for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
|
||||
},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_document_content_for_claude():
|
||||
"""Cache control should be preserved in document content for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "application/pdf",
|
||||
"data": "JVBERi0xLjQKJeLjz9MK",
|
||||
},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_tool_result_for_claude():
|
||||
"""Cache control should be preserved in tool_result for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01234",
|
||||
"content": "Tool result content",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
tool_message = next(msg for msg in result if msg.get("role") == "tool")
|
||||
assert tool_message["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_not_preserved_in_tool_result_for_non_claude():
|
||||
"""Cache control should NOT be preserved in tool_result for non-Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01234",
|
||||
"content": "Tool result content",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL
|
||||
)
|
||||
|
||||
tool_message = next(msg for msg in result if msg.get("role") == "tool")
|
||||
assert "cache_control" not in tool_message
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_assistant_text_for_claude():
|
||||
"""Cache control should be preserved in assistant text blocks for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthopicMessagesAssistantMessageParam(
|
||||
role="assistant",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Assistant response",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["role"] == "assistant"
|
||||
# When cache_control is present, content should be a list
|
||||
assert isinstance(result[0]["content"], list)
|
||||
assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_tool_use_for_claude():
|
||||
"""Cache control should be preserved in tool_use blocks for Claude models."""
|
||||
anthropic_messages = [
|
||||
AnthopicMessagesAssistantMessageParam(
|
||||
role="assistant",
|
||||
content=[
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01234",
|
||||
"name": "get_weather",
|
||||
"input": {"location": "Boston"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "tool_calls" in result[0]
|
||||
assert result[0]["tool_calls"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_tools_for_claude():
|
||||
"""Cache control should be preserved in tools for Claude models."""
|
||||
tools = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"input_schema": {"type": "object", "properties": {"location": {"type": "string"}}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_tools_to_openai(
|
||||
tools=tools, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_not_preserved_in_tools_for_non_claude():
|
||||
"""Cache control should NOT be preserved in tools for non-Claude models."""
|
||||
tools = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"input_schema": {"type": "object", "properties": {"location": {"type": "string"}}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_tools_to_openai(
|
||||
tools=tools, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "cache_control" not in result[0]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue