diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 1706f045f14..58120bab1f5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -168,6 +168,24 @@ class LiteLLMAnthropicMessagesAdapter: return provider_specific_fields.get("signature") return None + def _add_cache_control_if_applicable( + self, + source: Dict[str, Any], + target: Dict[str, Any], + model: Optional[str], + ) -> None: + """ + Extract cache_control from source and add to target if it should be preserved. + + Args: + source: Dict containing potential cache_control field + target: Dict to add cache_control to + model: Model name to check if cache_control should be preserved + """ + cache_control = source.get("cache_control") + if cache_control and model and self.is_anthropic_claude_model(model): + target["cache_control"] = cache_control + def translatable_anthropic_params(self) -> List: """ Which anthropic params, we need to translate to the openai format. @@ -202,15 +220,11 @@ class LiteLLMAnthropicMessagesAdapter: elif message_content and isinstance(message_content, list): for content in message_content: if content.get("type") == "text": - text_obj = ChatCompletionTextObject( + text_obj: Dict[str, Any] = ChatCompletionTextObject( type="text", text=content.get("text", "") ) - # Preserve cache_control if present (for prompt caching) - # Only for Anthropic models that support prompt caching - cache_control = content.get("cache_control") - if cache_control and model and self.is_anthropic_claude_model(model): - text_obj["cache_control"] = cache_control # type: ignore - new_user_content_list.append(text_obj) + self._add_cache_control_if_applicable(content, text_obj, model) + new_user_content_list.append(text_obj) # type: ignore elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format source = content.get("source", {}) @@ -222,25 +236,44 @@ class LiteLLMAnthropicMessagesAdapter: image_url_obj = ChatCompletionImageUrlObject( url=openai_image_url ) - image_obj = ChatCompletionImageObject( + image_obj: Dict[str, Any] = ChatCompletionImageObject( type="image_url", image_url=image_url_obj ) - new_user_content_list.append(image_obj) + self._add_cache_control_if_applicable(content, image_obj, model) + new_user_content_list.append(image_obj) # type: ignore + elif content.get("type") == "document": + # Convert Anthropic document format (PDF, etc.) to OpenAI format + source = content.get("source", {}) + openai_image_url = ( + self._translate_anthropic_image_to_openai(source) + ) + + if openai_image_url: + image_url_obj = ChatCompletionImageUrlObject( + url=openai_image_url + ) + doc_obj: Dict[str, Any] = ChatCompletionImageObject( + type="image_url", image_url=image_url_obj + ) + self._add_cache_control_if_applicable(content, doc_obj, model) + new_user_content_list.append(doc_obj) # type: ignore elif content.get("type") == "tool_result": if "content" not in content: - tool_result = ChatCompletionToolMessage( + tool_result: Dict[str, Any] = ChatCompletionToolMessage( role="tool", tool_call_id=content.get("tool_use_id", ""), content="", ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(content.get("content"), str): tool_result = ChatCompletionToolMessage( role="tool", tool_call_id=content.get("tool_use_id", ""), content=str(content.get("content", "")), ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(content.get("content"), list): # Combine all content items into a single tool message # to avoid creating multiple tool_result blocks with the same ID @@ -256,7 +289,8 @@ class LiteLLMAnthropicMessagesAdapter: tool_call_id=content.get("tool_use_id", ""), content=c, ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(c, dict): if c.get("type") == "text": tool_result = ChatCompletionToolMessage( @@ -266,7 +300,8 @@ class LiteLLMAnthropicMessagesAdapter: ), content=c.get("text", ""), ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] elif c.get("type") == "image": source = c.get("source", {}) openai_image_url = ( @@ -282,7 +317,8 @@ class LiteLLMAnthropicMessagesAdapter: ), content=openai_image_url, ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] else: # For multiple content items, combine into a single tool message # with list content to preserve all items while having one tool_use_id @@ -331,7 +367,8 @@ class LiteLLMAnthropicMessagesAdapter: tool_call_id=content.get("tool_use_id", ""), content=combined_content_parts, # type: ignore ) - tool_message_list.append(tool_result) + self._add_cache_control_if_applicable(content, tool_result, model) + tool_message_list.append(tool_result) # type: ignore[arg-type] if len(tool_message_list) > 0: new_messages.extend(tool_message_list) @@ -344,7 +381,9 @@ class LiteLLMAnthropicMessagesAdapter: ## ASSISTANT MESSAGE ## assistant_message_str: Optional[str] = None - tool_calls: List[ChatCompletionAssistantToolCall] = [] + assistant_content_list: List[Dict[str, Any]] = [] # For content blocks with cache_control + has_cache_control_in_text = False + tool_calls: List[Dict[str, Any]] = [] thinking_blocks: List[ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] ] = [] @@ -357,10 +396,14 @@ class LiteLLMAnthropicMessagesAdapter: assistant_message_str = str(content) elif isinstance(content, dict): if content.get("type") == "text": - if assistant_message_str is None: - assistant_message_str = content.get("text", "") - else: - assistant_message_str += content.get("text", "") + text_block: Dict[str, Any] = { + "type": "text", + "text": content.get("text", ""), + } + self._add_cache_control_if_applicable(content, text_block, model) + if "cache_control" in text_block: + has_cache_control_in_text = True + assistant_content_list.append(text_block) elif content.get("type") == "tool_use": function_chunk: ChatCompletionToolCallFunctionChunk = { "name": content.get("name", ""), @@ -384,13 +427,13 @@ class LiteLLMAnthropicMessagesAdapter: provider_specific_fields ) - tool_calls.append( - ChatCompletionAssistantToolCall( - id=content.get("id", ""), - type="function", - function=function_chunk, - ) + tool_call: Dict[str, Any] = ChatCompletionAssistantToolCall( + id=content.get("id", ""), + type="function", + function=function_chunk, ) + self._add_cache_control_if_applicable(content, tool_call, model) + tool_calls.append(tool_call) elif content.get("type") == "thinking": thinking_block = ChatCompletionThinkingBlock( type="thinking", @@ -411,18 +454,30 @@ class LiteLLMAnthropicMessagesAdapter: if ( assistant_message_str is not None + or len(assistant_content_list) > 0 or len(tool_calls) > 0 or len(thinking_blocks) > 0 ): + # Use list format if any text block has cache_control, otherwise use string + if has_cache_control_in_text and len(assistant_content_list) > 0: + assistant_content: Any = assistant_content_list + elif len(assistant_content_list) > 0 and not has_cache_control_in_text: + # Concatenate text blocks into string when no cache_control + assistant_content = "".join( + block.get("text", "") for block in assistant_content_list + ) + else: + assistant_content = assistant_message_str + assistant_message = ChatCompletionAssistantMessage( role="assistant", - content=assistant_message_str, + content=assistant_content, thinking_blocks=( thinking_blocks if len(thinking_blocks) > 0 else None ), ) if len(tool_calls) > 0: - assistant_message["tool_calls"] = tool_calls + assistant_message["tool_calls"] = tool_calls # type: ignore if len(thinking_blocks) > 0: assistant_message["thinking_blocks"] = thinking_blocks # type: ignore new_messages.append(assistant_message) @@ -532,10 +587,10 @@ class LiteLLMAnthropicMessagesAdapter: ) def translate_anthropic_tools_to_openai( - self, tools: List[AllAnthropicToolsValues] + self, tools: List[AllAnthropicToolsValues], model: Optional[str] = None ) -> List[ChatCompletionToolParam]: new_tools: List[ChatCompletionToolParam] = [] - mapped_tool_params = ["name", "input_schema", "description"] + mapped_tool_params = ["name", "input_schema", "description", "cache_control"] for tool in tools: function_chunk = ChatCompletionToolParamFunctionChunk( name=tool["name"], @@ -548,11 +603,11 @@ class LiteLLMAnthropicMessagesAdapter: for k, v in tool.items(): if k not in mapped_tool_params: # pass additional computer kwargs function_chunk.setdefault("parameters", {}).update({k: v}) - new_tools.append( - ChatCompletionToolParam(type="function", function=function_chunk) - ) + tool_param: Dict[str, Any] = ChatCompletionToolParam(type="function", function=function_chunk) + self._add_cache_control_if_applicable(tool, tool_param, model) + new_tools.append(tool_param) # type: ignore[arg-type] - return new_tools + return new_tools # type: ignore[return-value] def translate_anthropic_output_format_to_openai( self, output_format: Any @@ -621,10 +676,29 @@ class LiteLLMAnthropicMessagesAdapter: if "system" in anthropic_message_request: system_content = anthropic_message_request["system"] if system_content: - new_messages.insert( - 0, - ChatCompletionSystemMessage(role="system", content=system_content), - ) + # Handle system as string or array of content blocks + if isinstance(system_content, str): + new_messages.insert( + 0, + ChatCompletionSystemMessage(role="system", content=system_content), + ) + elif isinstance(system_content, list): + # Convert Anthropic system content blocks to OpenAI format + openai_system_content: List[Dict[str, Any]] = [] + model_name = anthropic_message_request.get("model", "") + for block in system_content: + if isinstance(block, dict) and block.get("type") == "text": + text_block: Dict[str, Any] = { + "type": "text", + "text": block.get("text", ""), + } + self._add_cache_control_if_applicable(block, text_block, model_name) + openai_system_content.append(text_block) + if openai_system_content: + new_messages.insert( + 0, + ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore + ) new_kwargs: ChatCompletionRequest = { "model": anthropic_message_request["model"], @@ -655,7 +729,8 @@ class LiteLLMAnthropicMessagesAdapter: tools = anthropic_message_request["tools"] if tools: new_kwargs["tools"] = self.translate_anthropic_tools_to_openai( - tools=cast(List[AllAnthropicToolsValues], tools) + tools=cast(List[AllAnthropicToolsValues], tools), + model=new_kwargs.get("model"), ) ## CONVERT THINKING @@ -827,7 +902,7 @@ class LiteLLMAnthropicMessagesAdapter: ) # extract usage usage: Usage = getattr(response, "usage") - anthropic_usage = AnthropicUsage( + anthropic_usage: Dict[str, Any] = AnthropicUsage( input_tokens=usage.prompt_tokens or 0, output_tokens=usage.completion_tokens or 0, ) @@ -843,7 +918,7 @@ class LiteLLMAnthropicMessagesAdapter: role="assistant", model=response.model or "unknown-model", stop_sequence=None, - usage=anthropic_usage, + usage=anthropic_usage, # type: ignore content=anthropic_content, # type: ignore stop_reason=anthropic_finish_reason, ) @@ -980,7 +1055,7 @@ class LiteLLMAnthropicMessagesAdapter: else: litellm_usage_chunk = None if litellm_usage_chunk is not None: - usage_delta = UsageDelta( + usage_delta: Dict[str, Any] = UsageDelta( input_tokens=litellm_usage_chunk.prompt_tokens or 0, output_tokens=litellm_usage_chunk.completion_tokens or 0, ) @@ -992,7 +1067,7 @@ class LiteLLMAnthropicMessagesAdapter: else: usage_delta = UsageDelta(input_tokens=0, output_tokens=0) return MessageBlockDelta( - type="message_delta", delta=delta, usage=usage_delta + type="message_delta", delta=delta, usage=usage_delta # type: ignore ) ( type_of_content, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 6aadbc058d1..c26d057fbf1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1108,3 +1108,309 @@ def test_streaming_chunk_with_both_text_and_tool_calls_issue_18238(): assert block_type == "tool_use" assert content_block_start["name"] == "Bash" assert content_block_start["id"] == "toolu_bdrk_013xRVejhv3ybmLEGCoZib2b" + + +# ============================================================================ +# Cache Control Transformation Tests +# ============================================================================ + +# Model constant for cache control tests +CACHE_CONTROL_BEDROCK_CONVERSE_MODEL = "bedrock/converse/global.anthropic.claude-opus-4-5-20251101-v1:0" +CACHE_CONTROL_NON_ANTHROPIC_MODEL = "gpt-4" + + +def test_should_add_cache_control_for_anthropic_model(): + """Should add cache_control to target for Anthropic Claude models.""" + adapter = LiteLLMAnthropicMessagesAdapter() + cache_control = {"type": "ephemeral"} + + for model in [ + CACHE_CONTROL_BEDROCK_CONVERSE_MODEL, + "anthropic/claude-sonnet-4-5", + "claude-opus-4-5-20251101", + "vertex_ai/claude-3-sonnet@20240229", + ]: + target = {} + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) + assert "cache_control" in target + assert target["cache_control"] == cache_control + + +def test_should_not_add_cache_control_for_non_anthropic_model(): + """Should not add cache_control for non-Anthropic models.""" + adapter = LiteLLMAnthropicMessagesAdapter() + cache_control = {"type": "ephemeral"} + + for model in [CACHE_CONTROL_NON_ANTHROPIC_MODEL, "openai/gpt-4-turbo", "gemini-pro"]: + target = {} + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) + assert "cache_control" not in target + + +def test_should_not_add_cache_control_when_none(): + """Should not add cache_control when source has None or empty cache_control.""" + adapter = LiteLLMAnthropicMessagesAdapter() + + for source in [{"cache_control": None}, {"cache_control": {}}, {"cache_control": ""}, {}]: + target = {} + adapter._add_cache_control_if_applicable(source, target, CACHE_CONTROL_BEDROCK_CONVERSE_MODEL) + assert "cache_control" not in target + + +def test_should_not_add_cache_control_when_model_none(): + """Should not add cache_control when model is None or empty.""" + adapter = LiteLLMAnthropicMessagesAdapter() + cache_control = {"type": "ephemeral"} + + for model in [None, ""]: + target = {} + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) + assert "cache_control" not in target + + +def test_cache_control_preserved_in_text_content_for_claude(): + """Cache control should be preserved in text content for Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "text", + "text": "This is cached content", + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_not_preserved_for_non_claude_model(): + """Cache control should NOT be preserved for non-Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "text", + "text": "This is cached content", + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL + ) + + assert len(result) == 1 + assert "cache_control" not in result[0]["content"][0] + + +def test_cache_control_preserved_in_image_content_for_claude(): + """Cache control should be preserved in image content for Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_preserved_in_document_content_for_claude(): + """Cache control should be preserved in document content for Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQKJeLjz9MK", + }, + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_preserved_in_tool_result_for_claude(): + """Cache control should be preserved in tool_result for Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "tool_result", + "tool_use_id": "toolu_01234", + "content": "Tool result content", + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + tool_message = next(msg for msg in result if msg.get("role") == "tool") + assert tool_message["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_not_preserved_in_tool_result_for_non_claude(): + """Cache control should NOT be preserved in tool_result for non-Claude models.""" + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[ + { + "type": "tool_result", + "tool_use_id": "toolu_01234", + "content": "Tool result content", + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL + ) + + tool_message = next(msg for msg in result if msg.get("role") == "tool") + assert "cache_control" not in tool_message + + +def test_cache_control_preserved_in_assistant_text_for_claude(): + """Cache control should be preserved in assistant text blocks for Claude models.""" + anthropic_messages = [ + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + { + "type": "text", + "text": "Assistant response", + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert result[0]["role"] == "assistant" + # When cache_control is present, content should be a list + assert isinstance(result[0]["content"], list) + assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_preserved_in_tool_use_for_claude(): + """Cache control should be preserved in tool_use blocks for Claude models.""" + anthropic_messages = [ + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + { + "type": "tool_use", + "id": "toolu_01234", + "name": "get_weather", + "input": {"location": "Boston"}, + "cache_control": {"type": "ephemeral"}, + } + ], + ) + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_messages_to_openai( + messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert "tool_calls" in result[0] + assert result[0]["tool_calls"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_preserved_in_tools_for_claude(): + """Cache control should be preserved in tools for Claude models.""" + tools = [ + { + "name": "get_weather", + "description": "Get weather for a location", + "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_tools_to_openai( + tools=tools, model=CACHE_CONTROL_BEDROCK_CONVERSE_MODEL + ) + + assert len(result) == 1 + assert result[0]["cache_control"] == {"type": "ephemeral"} + + +def test_cache_control_not_preserved_in_tools_for_non_claude(): + """Cache control should NOT be preserved in tools for non-Claude models.""" + tools = [ + { + "name": "get_weather", + "description": "Get weather for a location", + "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + adapter = LiteLLMAnthropicMessagesAdapter() + result = adapter.translate_anthropic_tools_to_openai( + tools=tools, model=CACHE_CONTROL_NON_ANTHROPIC_MODEL + ) + + assert len(result) == 1 + assert "cache_control" not in result[0]