diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index da120cad0e1..64c7ac62d03 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -13,7 +13,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.litellm_core_utils.tool_fields import ( ROUTES_WITHOUT_MCP_SERVERS, @@ -676,6 +676,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data, custom_llm_provider=target_provider, api_base=api_base if isinstance(api_base, str) else None, + deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")), drop_params=drop_params, ) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5379dcd6d68..907d4021fd3 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -237,6 +237,29 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) +def _is_azure_responses_target( + model: str | None, + custom_llm_provider: str | None, + deployment_model_info: Mapping[str, object] | None, +) -> bool: + if not model: + return False + if custom_llm_provider not in (None, LlmProviders.AZURE.value): + return False + if custom_llm_provider is None and not model.startswith("azure/"): + return False + + from litellm.responses.bridge_check import responses_api_bridge_check + + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + model_info, _ = responses_api_bridge_check( + model=resolved_model, + custom_llm_provider=provider, + deployment_model_info=deployment_model_info, + ) + return model_info.get("mode") == "responses" + + def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam: eager_input_streaming: Final = eager_input_streaming_flag(tool) if eager_input_streaming is None: @@ -272,6 +295,7 @@ class AnthropicAdapter: *, custom_llm_provider: str | None = None, api_base: str | None = None, + deployment_model_info: Mapping[str, object] | None = None, drop_params: bool = True, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ @@ -307,6 +331,7 @@ class AnthropicAdapter: anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, api_base=api_base, + deployment_model_info=deployment_model_info, drop_params=drop_params, report_reasoning_drops=True, ) @@ -502,10 +527,15 @@ class LiteLLMAnthropicMessagesAdapter: model: str | None = None, *, custom_llm_provider: str | None = None, + deployment_model_info: Mapping[str, object] | None = None, preserve_midturn_system: bool = False, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] - replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + replayable_messages: Final = ( + messages + if _is_azure_responses_target(model, custom_llm_provider, deployment_model_info) + else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + ) leading_count: Final = next( (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), len(replayable_messages), @@ -1289,6 +1319,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, api_base: str | None = None, + deployment_model_info: Mapping[str, object] | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, report_reasoning_drops: bool = False, @@ -1314,6 +1345,7 @@ class LiteLLMAnthropicMessagesAdapter: messages=messages_list, model=anthropic_message_request.get("model"), custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, preserve_midturn_system=preserve_midturn_system, ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 8a924dd32d6..3296163c64e 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2716,3 +2716,120 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s assert first["tools"] == [tool] assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"]) +@pytest.mark.parametrize( + "model,model_info,responses_target", + [ + ("azure/responses/qa-deployment", None, True), + ("azure/qa-deployment", {"mode": "responses"}, True), + ("azure/qa-deployment", None, False), + ("azure/qa-deployment", {"mode": "chat"}, False), + ], + ids=["responses-prefix", "responses-pin", "chat-deployment", "chat-pin"], +) +async def test_azure_messages_replay_encrypted_reasoning_only_to_responses( + monkeypatch: pytest.MonkeyPatch, + model: str, + model_info: dict[str, str] | None, + responses_target: bool, + stream: bool, + is_async: bool, +) -> None: + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "azure/qa-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supports_native_streaming": True, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + litellm.in_memory_llm_clients_cache.flush_cache() + chat_reply: Final = { + "id": "chatcmpl-azure", + "object": "chat.completion", + "created": 1, + "model": "qa-deployment", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello there"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + chat_stream: Final = ( + 'data: {"id":"chatcmpl-azure","object":"chat.completion.chunk","created":1,' + '"model":"qa-deployment","choices":[{"index":0,"delta":{"role":"assistant","content":"hello there"},' + '"finish_reason":null}]}\n\n' + 'data: {"id":"chatcmpl-azure","object":"chat.completion.chunk","created":1,' + '"model":"qa-deployment","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],' + '"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}\n\n' + "data: [DONE]\n\n" + ).encode() + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.route(host="qa-resource.openai.azure.com").mock( + return_value=( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_sse_body(_RESPONSES_SSE_EVENTS) if responses_target else chat_stream, + ) + if stream + else httpx.Response(200, json=_RESPONSES_COMPLETED_BODY if responses_target else chat_reply) + ) + ) + create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create + pending: Final = create( + model=model, + model_info=model_info, + max_tokens=100, + api_base="https://qa-resource.openai.azure.com/openai/v1", + api_version="v1", + api_key="test-api-key", + stream=stream, + messages=[ + {"role": "user", "content": "Use calc for 17*23."}, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Use the calculator.", + "signature": encrypted_reasoning_signature("qa-opaque-reasoning"), + }, + {"type": "tool_use", "id": "call_1", "name": "calc", "input": {"x": 391}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "391"}]}, + ], + ) + result: Final = await pending if is_async else pending + if stream: + chunks: Final = [chunk async for chunk in result] if hasattr(result, "__aiter__") else list(result) + assert "message_stop" in "".join(chunk.decode() if isinstance(chunk, bytes) else chunk for chunk in chunks) + else: + assert result["content"] == [{"type": "text", "text": "hello there"}] + body: Final = json.loads(route.calls.last.request.content) + if responses_target: + assert route.calls.last.request.url.path.endswith("/responses") + assert [item for item in body["input"] if item.get("type") == "reasoning"] == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Use the calculator."}], + "encrypted_content": "qa-opaque-reasoning", + } + ] + assert [item for item in body["input"] if item.get("type") == "function_call_output"] == [ + {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "391"}]} + ] + else: + assert route.calls.last.request.url.path.endswith("/chat/completions") + assert "qa-opaque-reasoning" not in json.dumps(body)