scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

fix(anthropic): preserve Azure Responses reasoning replay (#50024) (#50381)

Browse source 
Fixes #50024

Preserve encoded reasoning history through the existing Azure Responses bridge.
This commit is contained in:
ishaan-j-berri 2026-11-06 18:18:59 +08:00 • committed by GitHub
parent a12100b33b
commit 80194a7d70
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 152 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -13,7 +13,7 @@ from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.tool_fields import (
ROUTES_WITHOUT_MCP_SERVERS,
@ -676,6 +676,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data,
custom_llm_provider=target_provider,
api_base=api_base if isinstance(api_base, str) else None,
deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")),
drop_params=drop_params,
)

34
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -237,6 +237,29 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
def _is_azure_responses_target(
model: str | None,
custom_llm_provider: str | None,
deployment_model_info: Mapping[str, object] | None,
) -> bool:
if not model:
return False
if custom_llm_provider not in (None, LlmProviders.AZURE.value):
return False
if custom_llm_provider is None and not model.startswith("azure/"):
return False
from litellm.responses.bridge_check import responses_api_bridge_check
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
model_info, _ = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
deployment_model_info=deployment_model_info,
)
return model_info.get("mode") == "responses"
def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam:
eager_input_streaming: Final = eager_input_streaming_flag(tool)
if eager_input_streaming is None:
@ -272,6 +295,7 @@ class AnthropicAdapter:
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
@ -307,6 +331,7 @@ class AnthropicAdapter:
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=deployment_model_info,
drop_params=drop_params,
report_reasoning_drops=True,
)
@ -502,10 +527,15 @@ class LiteLLMAnthropicMessagesAdapter:
model: str | None = None,
*,
custom_llm_provider: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
preserve_midturn_system: bool = False,
) -> list[AllMessageValues]:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
replayable_messages: Final = (
messages
if _is_azure_responses_target(model, custom_llm_provider, deployment_model_info)
else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
)
leading_count: Final = next(
(i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
len(replayable_messages),
@ -1289,6 +1319,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
report_reasoning_drops: bool = False,
@ -1314,6 +1345,7 @@ class LiteLLMAnthropicMessagesAdapter:
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
preserve_midturn_system=preserve_midturn_system,
)
## ADD SYSTEM MESSAGE TO MESSAGES

117
tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py
View file

@ -2716,3 +2716,120 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
assert first["tools"] == [tool]
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"])
@pytest.mark.parametrize(
"model,model_info,responses_target",
[
("azure/responses/qa-deployment", None, True),
("azure/qa-deployment", {"mode": "responses"}, True),
("azure/qa-deployment", None, False),
("azure/qa-deployment", {"mode": "chat"}, False),
],
ids=["responses-prefix", "responses-pin", "chat-deployment", "chat-pin"],
)
async def test_azure_messages_replay_encrypted_reasoning_only_to_responses(
monkeypatch: pytest.MonkeyPatch,
model: str,
model_info: dict[str, str] | None,
responses_target: bool,
stream: bool,
is_async: bool,
) -> None:
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
"azure/qa-deployment": {
"litellm_provider": "azure",
"mode": "chat",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_native_streaming": True,
},
},
)
_invalidate_model_cost_lowercase_map()
litellm.in_memory_llm_clients_cache.flush_cache()
chat_reply: Final = {
"id": "chatcmpl-azure",
"object": "chat.completion",
"created": 1,
"model": "qa-deployment",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hello there"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
}
chat_stream: Final = (
'data: {"id":"chatcmpl-azure","object":"chat.completion.chunk","created":1,'
'"model":"qa-deployment","choices":[{"index":0,"delta":{"role":"assistant","content":"hello there"},'
'"finish_reason":null}]}\n\n'
'data: {"id":"chatcmpl-azure","object":"chat.completion.chunk","created":1,'
'"model":"qa-deployment","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],'
'"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}\n\n'
"data: [DONE]\n\n"
).encode()
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.route(host="qa-resource.openai.azure.com").mock(
return_value=(
httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content=_sse_body(_RESPONSES_SSE_EVENTS) if responses_target else chat_stream,
)
if stream
else httpx.Response(200, json=_RESPONSES_COMPLETED_BODY if responses_target else chat_reply)
)
)
create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create
pending: Final = create(
model=model,
model_info=model_info,
max_tokens=100,
api_base="https://qa-resource.openai.azure.com/openai/v1",
api_version="v1",
api_key="test-api-key",
stream=stream,
messages=[
{"role": "user", "content": "Use calc for 17*23."},
{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "Use the calculator.",
"signature": encrypted_reasoning_signature("qa-opaque-reasoning"),
},
{"type": "tool_use", "id": "call_1", "name": "calc", "input": {"x": 391}},
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "391"}]},
],
)
result: Final = await pending if is_async else pending
if stream:
chunks: Final = [chunk async for chunk in result] if hasattr(result, "__aiter__") else list(result)
assert "message_stop" in "".join(chunk.decode() if isinstance(chunk, bytes) else chunk for chunk in chunks)
else:
assert result["content"] == [{"type": "text", "text": "hello there"}]
body: Final = json.loads(route.calls.last.request.content)
if responses_target:
assert route.calls.last.request.url.path.endswith("/responses")
assert [item for item in body["input"] if item.get("type") == "reasoning"] == [
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "Use the calculator."}],
"encrypted_content": "qa-opaque-reasoning",
}
]
assert [item for item in body["input"] if item.get("type") == "function_call_output"] == [
{"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "391"}]}
]
else:
assert route.calls.last.request.url.path.endswith("/chat/completions")
assert "qa-opaque-reasoning" not in json.dumps(body)