From 4a5324534c460a28be76d97fa41741905187908b Mon Sep 17 00:00:00 2001 From: Devon Krisman Date: Tue, 18 Aug 2026 16:25:11 -0400 Subject: [PATCH] feat(anthropic-bridge): opt-in demotion or dropping of mid-turn system messages Clients like Claude Code send system-role reminder messages mid conversation. OpenAI accepts those anywhere, but OpenAI-compatible backends with strict chat templates (Qwen3 on vLLM) reject them with "System message must be at the beginning." Gated on LITELLM_DEMOTE_MIDTURN_SYSTEM: "true" (alias "demote") rewrites system entries after index 0 as user rows after the top-level system param is prepended; "drop" removes them entirely for backends where the reminder content is not wanted. Default behavior unchanged: upstream preserves mid-turn system rows deliberately (PR #34290), so this stays opt-in. --- .../adapters/transformation.py | 35 +- ...al_pass_through_adapters_transformation.py | 412 +++++++++--------- 2 files changed, 252 insertions(+), 195 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 34c2d837127..635fabab3de 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1,6 +1,7 @@ import copy import hashlib import json +import os from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast @@ -892,6 +893,37 @@ class LiteLLMAnthropicMessagesAdapter: text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj)) return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None + def _demote_midturn_system_messages( + self, + new_messages: list[AllMessageValues], # mutable-ok: matches ChatCompletionRequest.messages + ) -> list[AllMessageValues]: # mutable-ok: ChatCompletionRequest.messages requires a list + """Return the messages with system entries after index 0 rewritten or removed, opt-in. + + Gated on LITELLM_DEMOTE_MIDTURN_SYSTEM: "true" (or "demote") rewrites + each in-sequence system row as a user row, "drop" removes them + entirely, anything else leaves the messages untouched. OpenAI accepts + system messages anywhere in the conversation, but many + OpenAI-compatible backends enforce chat templates that reject + non-leading system rows (e.g. Qwen3 served by vLLM: "System message + must be at the beginning."). Clients like Claude Code send mid-turn + system reminders, so without this those requests 400. Demoting to a + user row mirrors how such reminders were historically delivered; + dropping trades their content for a prompt the backend caches better. + """ + mode: Final = os.environ.get("LITELLM_DEMOTE_MIDTURN_SYSTEM", "").strip().lower() + if mode not in ("true", "demote", "drop"): + return new_messages + if mode == "drop": + return [ # mutable-ok: ChatCompletionRequest.messages requires a list + message for index, message in enumerate(new_messages) if index == 0 or message.get("role") != "system" + ] + return [ # mutable-ok: ChatCompletionRequest.messages requires a list + ChatCompletionUserMessage(role="user", content=message.get("content") or "") + if index > 0 and message.get("role") == "system" + else message + for index, message in enumerate(new_messages) + ] + def _add_system_message_to_messages( self, new_messages: list[AllMessageValues], @@ -1135,10 +1167,11 @@ class LiteLLMAnthropicMessagesAdapter: ) ## ADD SYSTEM MESSAGE TO MESSAGES self._add_system_message_to_messages(new_messages, anthropic_message_request) + final_messages: Final = self._demote_midturn_system_messages(new_messages) new_kwargs: Final[ChatCompletionRequest] = { "model": anthropic_message_request["model"], - "messages": new_messages, + "messages": final_messages, } ## CONVERT METADATA (user_id + litellm metadata) self._translate_metadata_to_openai( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index e4dacc308dc..2c299ea27e1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -50,9 +50,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_content_block(): tool_calls=[ ChatCompletionDeltaToolCall( id="call_d581d130-e234-4315-94e8-27e7ff7c4e55", - function=Function( - arguments='{"location": "Boston"}', name="get_weather" - ), + function=Function(arguments='{"location": "Boston"}', name="get_weather"), type="function", index=0, ) @@ -66,9 +64,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_content_block(): ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) print(content_block_start) @@ -98,9 +94,7 @@ def test_translate_streaming_openai_chunk_strips_gemini_thought_from_tool_call_i tool_calls=[ ChatCompletionDeltaToolCall( id=combined, - function=Function( - arguments='{"a": 17, "b": 25}', name="add_numbers" - ), + function=Function(arguments='{"a": 17, "b": 25}', name="add_numbers"), type="function", index=0, ) @@ -114,9 +108,7 @@ def test_translate_streaming_openai_chunk_strips_gemini_thought_from_tool_call_i ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "tool_use" assert content_block_start["id"] == base @@ -161,9 +153,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_thinking_content_block(): ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "thinking" assert content_block_start == { @@ -199,9 +189,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_reasoning_content_only_co ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "thinking" assert content_block_start == { @@ -247,9 +235,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_thinking_signature_block( ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "thinking" assert content_block_start == { @@ -302,9 +288,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_content_block_thinking_an ( block_type, content_block_start, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "thinking" @@ -347,10 +331,7 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks(): assert "thinking_blocks" in result[1] assert len(result[1]["thinking_blocks"]) == 2 assert result[1]["thinking_blocks"][0]["type"] == "thinking" - assert ( - result[1]["thinking_blocks"][0]["thinking"] - == "I will call the get_weather tool." - ) + assert result[1]["thinking_blocks"][0]["thinking"] == "I will call the get_weather tool." assert result[1]["thinking_blocks"][0]["signature"] == "sigsig" assert result[1]["thinking_blocks"][1]["type"] == "redacted_thinking" assert result[1]["thinking_blocks"][1]["data"] == "REDACTED" @@ -411,9 +392,7 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): assert tool_message_idx is not None, "Tool message not found" assert user_message_idx is not None, "User message not found" - assert ( - tool_message_idx < user_message_idx - ), "Tool message should be placed before user message" + assert tool_message_idx < user_message_idx, "Tool message should be placed before user message" @pytest.mark.parametrize( @@ -722,6 +701,158 @@ def test_translate_anthropic_to_openai_without_metadata_sets_neither_user_nor_pr assert "prompt_cache_key" not in openai_request +@pytest.mark.parametrize("env_value", ["true", "demote"]) +def test_translate_anthropic_to_openai_demotes_midturn_system_when_enabled( + monkeypatch, + env_value: str, +): + """ + With LITELLM_DEMOTE_MIDTURN_SYSTEM=true (alias "demote"), in-sequence system rows are + rewritten as user rows so chat templates that reject non-leading system messages + (e.g. Qwen3 on vLLM) accept the request. The hoisted top-level prompt at index 0 + keeps `role: "system"`. + """ + monkeypatch.setenv("LITELLM_DEMOTE_MIDTURN_SYSTEM", env_value) + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "system": "Trusted top-level prompt.", + "messages": [ + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer."}, + {"role": "system", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": "Trusted top-level prompt."}, + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer.", "thinking_blocks": None}, + {"role": "user", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ] + + +def test_translate_anthropic_to_openai_demotes_midturn_system_block_content(monkeypatch): + """Demoted rows keep their translated content-block list, including cache_control.""" + monkeypatch.setenv("LITELLM_DEMOTE_MIDTURN_SYSTEM", "true") + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "First question."}, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Use the corrected result.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "user", "content": "First question."}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Use the corrected result.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ] + + +def test_translate_anthropic_to_openai_drops_midturn_system_when_requested(monkeypatch): + """ + With LITELLM_DEMOTE_MIDTURN_SYSTEM=drop, in-sequence system rows are removed entirely; + the hoisted top-level prompt at index 0 keeps `role: "system"` and every other turn + is untouched. + """ + monkeypatch.setenv("LITELLM_DEMOTE_MIDTURN_SYSTEM", "drop") + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "system": "Trusted top-level prompt.", + "messages": [ + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer."}, + {"role": "system", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": "Trusted top-level prompt."}, + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer.", "thinking_blocks": None}, + {"role": "user", "content": "Continue."}, + ] + + +def test_translate_anthropic_to_openai_drop_keeps_leading_system_row(monkeypatch): + """Without a top-level system param, a system row already at index 0 survives drop mode.""" + monkeypatch.setenv("LITELLM_DEMOTE_MIDTURN_SYSTEM", "drop") + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "messages": [ + {"role": "system", "content": "Leading system row."}, + {"role": "user", "content": "First question."}, + {"role": "system", "content": "Use the corrected result."}, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": "Leading system row."}, + {"role": "user", "content": "First question."}, + ] + + +@pytest.mark.parametrize("env_value", ["", "false", "1", "TRUE_", "drop_"]) +def test_translate_anthropic_to_openai_midturn_system_preserved_unless_opted_in( + monkeypatch, + env_value: str, +): + """Anything other than "true" keeps the default in-place behavior.""" + monkeypatch.setenv("LITELLM_DEMOTE_MIDTURN_SYSTEM", env_value) + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "First question."}, + {"role": "system", "content": "Use the corrected result."}, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "user", "content": "First question."}, + {"role": "system", "content": "Use the corrected result."}, + ] + + def test_translate_openai_content_to_anthropic_empty_function_arguments(): """Test that empty function arguments are handled safely and don't cause JSON parsing errors.""" @@ -735,7 +866,8 @@ def test_translate_openai_content_to_anthropic_empty_function_arguments(): id="call_empty_args", type="function", function=Function( - name="test_function", arguments="" # empty arguments string + name="test_function", + arguments="", # empty arguments string ), ) ], @@ -750,9 +882,7 @@ def test_translate_openai_content_to_anthropic_empty_function_arguments(): assert result[0]["type"] == "tool_use" assert result[0]["id"] == "call_empty_args" assert result[0]["name"] == "test_function" - assert ( - result[0]["input"] == {} - ), "Empty function arguments should result in empty dict" + assert result[0]["input"] == {}, "Empty function arguments should result in empty dict" def test_translate_openai_content_to_anthropic_text_and_tool_calls(): @@ -872,9 +1002,7 @@ def test_translate_openai_response_to_anthropic_text_and_tool_calls(): ChatCompletionAssistantToolCall( id="call_tool_combo", type="function", - function=Function( - name="get_weather", arguments='{"location": "Paris"}' - ), + function=Function(name="get_weather", arguments='{"location": "Paris"}'), ) ], ), @@ -884,9 +1012,7 @@ def test_translate_openai_response_to_anthropic_text_and_tool_calls(): ) adapter = LiteLLMAnthropicMessagesAdapter() - anthropic_response = adapter.translate_openai_response_to_anthropic( - response=openai_response - ) + anthropic_response = adapter.translate_openai_response_to_anthropic(response=openai_response) anthropic_content = anthropic_response.get("content") assert anthropic_content is not None @@ -927,9 +1053,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_with_partial_json(): ( type_of_content, content_block_delta, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic(choices=choices) print("Type of content:", type_of_content) print("Content block delta:", content_block_delta) @@ -1004,9 +1128,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_thinking_delta(): ( type_of_content, content_block_delta, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic(choices=choices) assert type_of_content == "thinking_delta" assert content_block_delta["type"] == "thinking_delta" @@ -1049,9 +1171,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_with_thinking(): ( type_of_content, content_block_delta, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic(choices=choices) assert type_of_content == "signature_delta" assert content_block_delta["type"] == "signature_delta" @@ -1115,9 +1235,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_emits_signature_when_thin ( block_type, content_block_start, - ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "thinking" @@ -1157,9 +1275,7 @@ def test_translate_anthropic_messages_to_openai_user_message_with_base64_image() # Check image content assert result[0]["content"][1]["type"] == "image_url" assert "image_url" in result[0]["content"][1] - assert result[0]["content"][1]["image_url"]["url"].startswith( - "data:image/png;base64," - ) + assert result[0]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,") assert ( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" in result[0]["content"][1]["image_url"]["url"] @@ -1197,18 +1313,14 @@ def test_translate_anthropic_messages_to_openai_user_message_with_url_image(): # Check image content assert result[0]["content"][1]["type"] == "image_url" assert "image_url" in result[0]["content"][1] - assert ( - result[0]["content"][1]["image_url"]["url"] == "https://example.com/forest.jpg" - ) + assert result[0]["content"][1]["image_url"]["url"] == "https://example.com/forest.jpg" def test_translate_anthropic_messages_to_openai_tool_result_with_base64_image(): """Test that base64 images in tool results are correctly translated to OpenAI format.""" anthropic_messages = [ - AnthropicMessagesUserMessageParam( - role="user", content=[{"type": "text", "text": "Take a screenshot"}] - ), + AnthropicMessagesUserMessageParam(role="user", content=[{"type": "text", "text": "Take a screenshot"}]), AnthopicMessagesAssistantMessageParam( role="assistant", content=[ @@ -1360,9 +1472,7 @@ def test_translate_anthropic_messages_to_openai_mixed_content_with_image(): # Check first image (base64) assert result[0]["content"][1]["type"] == "image_url" - assert result[0]["content"][1]["image_url"]["url"].startswith( - "data:image/png;base64," - ) + assert result[0]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,") # Check middle text assert result[0]["content"][2]["type"] == "text" @@ -1370,9 +1480,7 @@ def test_translate_anthropic_messages_to_openai_mixed_content_with_image(): # Check second image (URL) assert result[0]["content"][3]["type"] == "image_url" - assert ( - result[0]["content"][3]["image_url"]["url"] == "https://example.com/image2.jpg" - ) + assert result[0]["content"][3]["image_url"]["url"] == "https://example.com/image2.jpg" # Check final text assert result[0]["content"][4]["type"] == "text" @@ -1418,10 +1526,7 @@ def test_translate_anthropic_messages_to_openai_tool_use_with_signature(): assert tool_call["id"] == "call_386f67af31f9415781bc35071405" assert "function" in tool_call assert "provider_specific_fields" in tool_call["function"] - assert ( - tool_call["function"]["provider_specific_fields"]["thought_signature"] - == test_signature - ) + assert tool_call["function"]["provider_specific_fields"]["thought_signature"] == test_signature def test_translate_anthropic_messages_to_openai_tool_result_with_multiple_content_items(): @@ -1479,9 +1584,7 @@ def test_translate_anthropic_messages_to_openai_tool_result_with_multiple_conten result = adapter.translate_anthropic_messages_to_openai(messages=anthropic_messages) # Count how many tool messages have the same tool_call_id - tool_messages = [ - msg for msg in result if isinstance(msg, dict) and msg.get("role") == "tool" - ] + tool_messages = [msg for msg in result if isinstance(msg, dict) and msg.get("role") == "tool"] tool_call_ids = [msg.get("tool_call_id") for msg in tool_messages] # The critical assertion: each tool_call_id should appear only ONCE @@ -1497,12 +1600,8 @@ def test_translate_anthropic_messages_to_openai_tool_result_with_multiple_conten # The content should be a list with all items combined tool_message = tool_messages[0] assert tool_message["tool_call_id"] == "toolu_016hYHBkTf4JDF3p22UoYk5C" - assert isinstance( - tool_message["content"], list - ), "Multiple content items should be combined into a list" - assert ( - len(tool_message["content"]) == 3 - ), f"Expected 3 content items, got {len(tool_message['content'])}" + assert isinstance(tool_message["content"], list), "Multiple content items should be combined into a list" + assert len(tool_message["content"]) == 3, f"Expected 3 content items, got {len(tool_message['content'])}" # Verify content types assert tool_message["content"][0]["type"] == "text" @@ -1551,17 +1650,14 @@ def test_translate_anthropic_messages_to_openai_tool_result_single_item_backward adapter = LiteLLMAnthropicMessagesAdapter() result = adapter.translate_anthropic_messages_to_openai(messages=anthropic_messages) - tool_messages = [ - msg for msg in result if isinstance(msg, dict) and msg.get("role") == "tool" - ] + tool_messages = [msg for msg in result if isinstance(msg, dict) and msg.get("role") == "tool"] assert len(tool_messages) == 1 tool_message = tool_messages[0] # Single item should be a string for backward compatibility assert isinstance(tool_message["content"], str), ( - f"Single content item should be a string for backward compatibility, " - f"got {type(tool_message['content'])}" + f"Single content item should be a string for backward compatibility, got {type(tool_message['content'])}" ) assert tool_message["content"] == "72°F and sunny" @@ -1610,9 +1706,7 @@ def test_streaming_chunk_with_both_text_and_tool_calls_issue_18238(): ( block_type, content_block_start, - ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "tool_use" assert content_block_start["name"] == "Bash" @@ -1656,9 +1750,7 @@ def test_streaming_chunk_with_text_and_empty_tool_calls_returns_text_delta(): ( block_type, content_block_start, - ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices - ) + ) = adapter._translate_streaming_openai_chunk_to_anthropic_content_block(choices=choices) assert block_type == "text" assert content_block_start == {"type": "text", "text": ""} @@ -1669,15 +1761,12 @@ def test_streaming_chunk_with_text_and_empty_tool_calls_returns_text_delta(): # ============================================================================ # Model constant for cache control tests -CACHE_CONTROL_BEDROCK_CONVERSE_MODEL = ( - "bedrock/converse/global.anthropic.claude-opus-4-5-20251101-v1:0" -) +CACHE_CONTROL_BEDROCK_CONVERSE_MODEL = "bedrock/converse/global.anthropic.claude-opus-4-5-20251101-v1:0" CACHE_CONTROL_NON_ANTHROPIC_MODEL = "gpt-4" # Bedrock Application Inference Profile ARN: the string contains neither # "anthropic" nor "claude", so the model can only be recognized via its ARN shape CACHE_CONTROL_BEDROCK_ARN_MODEL = ( - "bedrock/converse/arn:aws:bedrock:us-east-1:123456789012:" - "application-inference-profile/abcdef123456" + "bedrock/converse/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456" ) @@ -1693,9 +1782,7 @@ def test_should_add_cache_control_for_anthropic_model(): "vertex_ai/claude-3-sonnet@20240229", ]: target = {} - adapter._add_cache_control_if_applicable( - {"cache_control": cache_control}, target, model - ) + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) assert "cache_control" in target assert target["cache_control"] == cache_control @@ -1711,9 +1798,7 @@ def test_should_not_add_cache_control_for_non_anthropic_model(): "gemini-pro", ]: target = {} - adapter._add_cache_control_if_applicable( - {"cache_control": cache_control}, target, model - ) + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) assert "cache_control" not in target @@ -1728,9 +1813,7 @@ def test_should_not_add_cache_control_when_none(): {}, ]: target = {} - adapter._add_cache_control_if_applicable( - source, target, CACHE_CONTROL_BEDROCK_CONVERSE_MODEL - ) + adapter._add_cache_control_if_applicable(source, target, CACHE_CONTROL_BEDROCK_CONVERSE_MODEL) assert "cache_control" not in target @@ -1741,9 +1824,7 @@ def test_should_not_add_cache_control_when_model_none(): for model in [None, ""]: target = {} - adapter._add_cache_control_if_applicable( - {"cache_control": cache_control}, target, model - ) + adapter._add_cache_control_if_applicable({"cache_control": cache_control}, target, model) assert "cache_control" not in target @@ -1849,12 +1930,7 @@ def test_cache_control_fix_does_not_broaden_claude_detection(): make is_anthropic_claude_model treat ARN profiles as Claude, which would route thinking params through unmodified and break non-Claude Bedrock profiles. """ - assert ( - LiteLLMAnthropicMessagesAdapter.is_anthropic_claude_model( - CACHE_CONTROL_BEDROCK_ARN_MODEL - ) - is False - ) + assert LiteLLMAnthropicMessagesAdapter.is_anthropic_claude_model(CACHE_CONTROL_BEDROCK_ARN_MODEL) is False def test_thinking_preserved_for_bedrock_arn_inference_profile(): @@ -2322,9 +2398,7 @@ def test_translate_streaming_openai_chunk_to_anthropic_reasoning_content_without ( type_of_content, content_block_delta, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( - choices=choices - ) + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic(choices=choices) assert type_of_content == "thinking_delta" assert content_block_delta["type"] == "thinking_delta" @@ -2356,9 +2430,7 @@ def test_translate_openai_response_to_anthropic_with_reasoning_content_only(): ) adapter = LiteLLMAnthropicMessagesAdapter() - anthropic_response = adapter.translate_openai_response_to_anthropic( - response=openai_response - ) + anthropic_response = adapter.translate_openai_response_to_anthropic(response=openai_response) anthropic_content = anthropic_response.get("content") assert anthropic_content is not None @@ -2371,9 +2443,7 @@ def test_translate_openai_response_to_anthropic_with_reasoning_content_only(): # Second block should be text assert anthropic_content[1]["type"] == "text" - assert ( - anthropic_content[1]["text"] == 'There are **3** "r"s in the word strawberry.' - ) + assert anthropic_content[1]["text"] == 'There are **3** "r"s in the word strawberry.' assert anthropic_response.get("stop_reason") == "end_turn" @@ -2425,9 +2495,7 @@ def test_truncate_tool_name_deterministic(): def test_truncate_tool_name_avoids_collisions(): """Similar long names should produce different truncated names.""" name1 = "process_user_data_with_validation_and_error_handling_for_production_environment" - name2 = ( - "process_user_data_with_validation_and_error_handling_for_staging_environment" - ) + name2 = "process_user_data_with_validation_and_error_handling_for_staging_environment" result1 = truncate_tool_name(name1) result2 = truncate_tool_name(name2) @@ -2447,9 +2515,7 @@ def test_create_tool_name_mapping_no_long_names(): def test_create_tool_name_mapping_with_long_names(): """Mapping should contain entries for truncated names.""" - long_name = ( - "a_very_long_tool_name_that_exceeds_the_64_character_limit_imposed_by_openai" - ) + long_name = "a_very_long_tool_name_that_exceeds_the_64_character_limit_imposed_by_openai" tools = [ {"name": "short_name"}, {"name": long_name}, @@ -2474,9 +2540,7 @@ def test_translate_anthropic_tools_with_long_names(): ] adapter = LiteLLMAnthropicMessagesAdapter() - result, tool_name_mapping = adapter.translate_anthropic_tools_to_openai( - tools=tools, model="gpt-4" - ) + result, tool_name_mapping = adapter.translate_anthropic_tools_to_openai(tools=tools, model="gpt-4") assert len(result) == 1 # The tool name should be truncated @@ -2498,9 +2562,7 @@ def test_translate_anthropic_tools_mixed_names(): ] adapter = LiteLLMAnthropicMessagesAdapter() - result, tool_name_mapping = adapter.translate_anthropic_tools_to_openai( - tools=tools, model="gpt-4" - ) + result, tool_name_mapping = adapter.translate_anthropic_tools_to_openai(tools=tools, model="gpt-4") assert len(result) == 2 # Short name unchanged @@ -2514,9 +2576,7 @@ def test_translate_anthropic_tools_mixed_names(): def test_translate_openai_response_restores_tool_names(): """Tool names in responses should be restored to original.""" - original_name = ( - "a_very_long_tool_name_that_needs_truncation_for_openai_api_compatibility" - ) + original_name = "a_very_long_tool_name_that_needs_truncation_for_openai_api_compatibility" truncated_name = truncate_tool_name(original_name) tool_name_mapping = {truncated_name: original_name} @@ -2548,9 +2608,7 @@ def test_translate_openai_response_restores_tool_names(): ) adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_openai_response_to_anthropic( - response=response, tool_name_mapping=tool_name_mapping - ) + result = adapter.translate_openai_response_to_anthropic(response=response, tool_name_mapping=tool_name_mapping) # Find the tool_use block in the response tool_use_blocks = [c for c in result["content"] if c.get("type") == "tool_use"] @@ -2716,9 +2774,7 @@ def test_translate_openai_usage_to_anthropic_cache_tokens_from_dict_details_with "cache_write_tokens": 20.0, } - anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta( - usage - ) + anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta(usage) assert anthropic_usage["input_tokens"] == 70 assert anthropic_usage["output_tokens"] == 50 @@ -2737,9 +2793,7 @@ def test_translate_openai_usage_to_anthropic_ignores_fractional_cache_tokens(): "cache_creation_tokens": 20.25, } - anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta( - usage - ) + anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta(usage) assert anthropic_usage["input_tokens"] == 120 assert anthropic_usage["output_tokens"] == 50 @@ -2756,9 +2810,7 @@ def test_translate_openai_usage_to_anthropic_ignores_bool_cache_tokens(): usage.cache_read_input_tokens = True usage.cache_creation_input_tokens = True - anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta( - usage - ) + anthropic_usage = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta(usage) assert anthropic_usage["input_tokens"] == 120 assert anthropic_usage["output_tokens"] == 50 @@ -2977,9 +3029,7 @@ def test_translate_streaming_openai_response_to_anthropic_cache_tokens_with_appl assert message_delta["usage"]["output_tokens"] == 50 assert message_delta["usage"]["cache_read_input_tokens"] == 30 assert message_delta["usage"]["cache_creation_input_tokens"] == 20 - assert message_delta["context_management"]["applied_edits"][0]["type"] == ( - "compact_20260112" - ) + assert message_delta["context_management"]["applied_edits"][0]["type"] == ("compact_20260112") # ===================================================================== @@ -3154,15 +3204,8 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert schema["required"] == ["user"] assert schema["properties"]["user"]["additionalProperties"] is False assert schema["properties"]["user"]["required"] == ["name", "address"] - assert ( - schema["properties"]["user"]["properties"]["address"][ - "additionalProperties" - ] - is False - ) - assert schema["properties"]["user"]["properties"]["address"]["required"] == [ - "city" - ] + assert schema["properties"]["user"]["properties"]["address"]["additionalProperties"] is False + assert schema["properties"]["user"]["properties"]["address"]["required"] == ["city"] def test_array_items_object_adds_additional_properties_false(self): output_format = { @@ -3237,19 +3280,9 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert sorted(schema["required"]) == ["age", "email", "name"] def test_invalid_output_format_returns_none(self): - assert ( - self.adapter.translate_anthropic_output_format_to_openai("invalid") is None - ) - assert ( - self.adapter.translate_anthropic_output_format_to_openai({"type": "text"}) - is None - ) - assert ( - self.adapter.translate_anthropic_output_format_to_openai( - {"type": "json_schema"} - ) - is None - ) + assert self.adapter.translate_anthropic_output_format_to_openai("invalid") is None + assert self.adapter.translate_anthropic_output_format_to_openai({"type": "text"}) is None + assert self.adapter.translate_anthropic_output_format_to_openai({"type": "json_schema"}) is None class TestAnthropicStreamWrapperToolArgs: @@ -3453,9 +3486,7 @@ def test_translate_openai_response_to_anthropic_with_polyfill_compaction_block() ) response = _make_simple_openai_response(text="Hello after compaction.") adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_openai_response_to_anthropic( - response=response, polyfill_result=polyfill - ) + result = adapter.translate_openai_response_to_anthropic(response=response, polyfill_result=polyfill) content = result.get("content") assert content is not None @@ -3487,9 +3518,7 @@ def test_translate_openai_response_to_anthropic_with_polyfill_iterations_usage() ) response = _make_simple_openai_response(prompt_tokens=100, completion_tokens=30) adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_openai_response_to_anthropic( - response=response, polyfill_result=polyfill - ) + result = adapter.translate_openai_response_to_anthropic(response=response, polyfill_result=polyfill) usage = result.get("usage") assert usage is not None @@ -3544,13 +3573,9 @@ def test_translate_openai_response_to_anthropic_with_polyfill_both_compaction_an {"type": "compaction", "input_tokens": 300, "output_tokens": 75}, ], ) - response = _make_simple_openai_response( - text="After compaction.", prompt_tokens=120, completion_tokens=40 - ) + response = _make_simple_openai_response(text="After compaction.", prompt_tokens=120, completion_tokens=40) adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_openai_response_to_anthropic( - response=response, polyfill_result=polyfill - ) + result = adapter.translate_openai_response_to_anthropic(response=response, polyfill_result=polyfill) # compaction block must come first content = result.get("content") @@ -3652,7 +3677,9 @@ def test_translate_anthropic_tools_to_openai_omits_unset_strict(): assert function["parameters"]["required"] == ["query"] -TOOL_RESULT_IMAGE_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +TOOL_RESULT_IMAGE_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) TOOL_RESULT_IMAGE_URL = "https://example.com/screenshot.png" @@ -3660,8 +3687,7 @@ def _anthropic_tool_use_turn(*tool_use_ids): return AnthopicMessagesAssistantMessageParam( role="assistant", content=[ - {"type": "tool_use", "id": tid, "name": "read_file", "input": {"path": "img.png"}} - for tid in tool_use_ids + {"type": "tool_use", "id": tid, "name": "read_file", "input": {"path": "img.png"}} for tid in tool_use_ids ], ) @@ -3781,9 +3807,7 @@ def test_tool_result_parallel_tool_calls_keep_tool_message_adjacency(): result = _run_chat_completions_pipeline( [ _anthropic_tool_use_turn("toolu_01", "toolu_02"), - _anthropic_tool_result_turn( - {"toolu_01": [_base64_image_block()], "toolu_02": [_url_image_block()]} - ), + _anthropic_tool_result_turn({"toolu_01": [_base64_image_block()], "toolu_02": [_url_image_block()]}), ] )