mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge pull request #40451 from BerriAI/litellm_responses_bridge_replay_encrypted_reasoning
fix(anthropic): replay OpenAI encrypted reasoning byte for byte through the /v1/messages bridge
This commit is contained in:
commit
8ec2f00955
19 changed files with 1004 additions and 83 deletions
|
|
@ -25,7 +25,7 @@ import litellm
|
|||
from litellm import ModelResponse
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
responses_reasoning_item_from_thinking_blocks,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
|
|
@ -129,8 +129,8 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:
|
|||
return stored
|
||||
raw_blocks: Final = msg.get("thinking_blocks") or ()
|
||||
blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json
|
||||
from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
|
||||
return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload
|
||||
replayed: Final = responses_reasoning_items_from_thinking_blocks(blocks)
|
||||
return [dict(item) for item in replayed] # mutable-ok: API message payload
|
||||
|
||||
|
||||
def _build_reasoning_item(
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import io
|
|||
import json
|
||||
import mimetypes
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from itertools import groupby
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
|
|
@ -1823,14 +1823,11 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
|
|||
return None, message_content
|
||||
|
||||
|
||||
def _readable_thinking_text(
|
||||
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
|
||||
) -> str:
|
||||
def _readable_thinking_text(block: Mapping[str, object]) -> str:
|
||||
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
|
||||
if block.get("type") != "thinking":
|
||||
return ""
|
||||
thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag
|
||||
return str(thinking or "")
|
||||
return str(block.get("thinking") or "")
|
||||
|
||||
|
||||
def reasoning_content_from_thinking_blocks(
|
||||
|
|
@ -1843,24 +1840,125 @@ def reasoning_content_from_thinking_blocks(
|
|||
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
|
||||
|
||||
|
||||
def responses_reasoning_item_from_thinking_blocks(
|
||||
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
|
||||
) -> ChatCompletionReasoningItem | None:
|
||||
"""Build a Responses API `reasoning` input item from Anthropic thinking blocks.
|
||||
ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:"
|
||||
|
||||
The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
|
||||
did not mint itself, while an item without an id is always accepted.
|
||||
|
||||
def encrypted_reasoning_signature(encrypted_content: str) -> str:
|
||||
"""The opaque value a Responses API reasoning item's `encrypted_content` travels in.
|
||||
|
||||
Anthropic clients echo a thinking block's `signature` and a redacted block's `data`
|
||||
back verbatim, so either field can carry the encrypted reasoning across turns; the
|
||||
prefix tells the two apart from a signature Anthropic minted.
|
||||
"""
|
||||
return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}"
|
||||
|
||||
|
||||
def _carries_encrypted_reasoning(signature: object) -> bool:
|
||||
return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX)
|
||||
|
||||
|
||||
def encrypted_content_from_signature(signature: object) -> str | None:
|
||||
if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature):
|
||||
return None
|
||||
return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None
|
||||
|
||||
|
||||
def _encrypted_reasoning_field(block: Mapping[str, object]) -> object:
|
||||
match block.get("type"):
|
||||
case "thinking":
|
||||
return block.get("signature")
|
||||
case "redacted_thinking":
|
||||
return block.get("data")
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
|
||||
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
|
||||
|
||||
|
||||
def is_encrypted_reasoning_block(block: object) -> bool:
|
||||
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
|
||||
|
||||
Only the Responses API that minted the content can read it back, so an Anthropic
|
||||
backend has to drop such a block rather than fail signature verification on it.
|
||||
"""
|
||||
if not isinstance(block, Mapping):
|
||||
return False
|
||||
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
|
||||
return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping))
|
||||
|
||||
|
||||
def strip_encrypted_reasoning_from_messages(messages: object) -> None:
|
||||
"""Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from
|
||||
Anthropic-shaped history.
|
||||
|
||||
The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a
|
||||
provider that did not mint the block rejects it signed (a foreign signature) and unsigned
|
||||
(a missing signature) alike, so keeping its text as an unsigned thinking block only moves
|
||||
the 400 from the router to the provider.
|
||||
|
||||
Mutates the content lists in place: the router's fallback snapshot shares these
|
||||
message objects, so a rebound list would replay the stripped blocks on the fallback hop.
|
||||
"""
|
||||
if not isinstance(messages, list):
|
||||
return
|
||||
for content in _anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json
|
||||
_strip_encrypted_reasoning_from_blocks(content)
|
||||
|
||||
|
||||
def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
|
||||
return (
|
||||
cast(list[object], content) # cast-ok: narrowed by isinstance
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
|
||||
if isinstance(content, list)
|
||||
)
|
||||
|
||||
|
||||
def _strip_encrypted_reasoning_from_blocks(content: object) -> None:
|
||||
blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance
|
||||
kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block))
|
||||
blocks[:] = kept # rebind-ok: shared with fallback snapshot
|
||||
|
||||
|
||||
def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
|
||||
index, block = indexed_block
|
||||
return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary"
|
||||
|
||||
|
||||
def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None:
|
||||
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
|
||||
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
|
||||
for block in thinking_blocks
|
||||
for block in group
|
||||
if (text := _readable_thinking_text(block))
|
||||
]
|
||||
encrypted_content: Final = encrypted_content_of_block(group[0])
|
||||
if encrypted_content is not None:
|
||||
return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content)
|
||||
if not summary:
|
||||
return None
|
||||
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
|
||||
|
||||
|
||||
def responses_reasoning_items_from_thinking_blocks(
|
||||
thinking_blocks: Iterable[Mapping[str, object]],
|
||||
) -> tuple[ChatCompletionReasoningItem, ...]:
|
||||
"""Build Responses API `reasoning` input items from Anthropic thinking blocks.
|
||||
|
||||
A block carrying encrypted reasoning replays the item it came from byte for byte;
|
||||
a run of plain thinking blocks collapses into one summary-only item. No item carries
|
||||
an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty
|
||||
one, while an item without an id is always accepted.
|
||||
"""
|
||||
return tuple(
|
||||
item
|
||||
for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key)
|
||||
if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None
|
||||
)
|
||||
|
||||
|
||||
def _parse_content_for_reasoning(
|
||||
message_text: str | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from litellm.types.utils import GenericImageParsingChunk
|
|||
from .common_utils import (
|
||||
convert_content_list_to_str,
|
||||
infer_content_type_from_url_and_content,
|
||||
is_encrypted_reasoning_block,
|
||||
is_non_content_values_set,
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
|
|
@ -2299,13 +2300,16 @@ def sanitize_messages_for_tool_calling(
|
|||
|
||||
|
||||
def _is_unsignable_thinking_block(block: object) -> bool:
|
||||
"""A `thinking` block that Anthropic cannot accept on input.
|
||||
"""A thinking block that Anthropic cannot accept on input.
|
||||
|
||||
Anthropic verifies the thinking signature cryptographically, so a block whose
|
||||
signature is null, empty, or missing (e.g. from an open-source reasoning model)
|
||||
is rejected with a 400 and must be dropped rather than blanked or repaired.
|
||||
`redacted_thinking` blocks carry no signature and are always kept.
|
||||
is rejected with a 400 and must be dropped rather than blanked or repaired, and
|
||||
so is a block whose signature or data carries another provider's encrypted
|
||||
reasoning. A `redacted_thinking` block Anthropic minted is always kept.
|
||||
"""
|
||||
if is_encrypted_reasoning_block(block):
|
||||
return True
|
||||
if not isinstance(block, dict) or block.get("type") != "thinking":
|
||||
return False
|
||||
signature: Final = block.get("signature")
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
|
|
@ -1201,6 +1202,32 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A
|
|||
return out
|
||||
|
||||
|
||||
def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape
|
||||
if not isinstance(message, Mapping):
|
||||
return message
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return message
|
||||
kept: Final = [b for b in content if not is_encrypted_reasoning_block(b)] # mutable-ok: API message payload
|
||||
if len(kept) == len(content):
|
||||
return message
|
||||
if not kept:
|
||||
return None
|
||||
return {**message, "content": kept} # mutable-ok: API message payload
|
||||
|
||||
|
||||
def strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
||||
messages: Sequence[dict], # mutable-ok: Anthropic message payload shape
|
||||
) -> list[dict]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
|
||||
"""
|
||||
Drop thinking / redacted_thinking blocks that carry another provider's encrypted
|
||||
reasoning (a turn the Responses API bridge served) before the request reaches
|
||||
Anthropic, which cannot verify them. Anthropic's own signed blocks are kept.
|
||||
"""
|
||||
stripped: Final = (_without_encrypted_reasoning_blocks(m) for m in messages)
|
||||
return [m for m in stripped if m is not None] # mutable-ok: API message payload
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
||||
data: dict[str, Any],
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
|
|||
from litellm.llms.anthropic.common_utils import (
|
||||
is_empty_unsigned_thinking_block,
|
||||
normalize_anthropic_tool_use_id,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.context_management import (
|
||||
PolyfillResult,
|
||||
|
|
@ -417,7 +418,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
model: str | None = None,
|
||||
) -> list:
|
||||
new_messages: Final[list[AllMessageValues]] = []
|
||||
for m in messages:
|
||||
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
|
||||
for m in replayable_messages:
|
||||
user_message: ChatCompletionUserMessage | None = None
|
||||
tool_message_list: list[ChatCompletionToolMessage] = []
|
||||
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from ...common_utils import (
|
|||
AnthropicModelInfo,
|
||||
optionally_handle_anthropic_oauth,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
||||
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
|
||||
|
|
@ -613,7 +614,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages = strip_advisor_blocks_from_messages(messages)
|
||||
|
||||
anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
|
||||
messages=messages,
|
||||
messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages),
|
||||
max_tokens=max_tokens,
|
||||
model=model,
|
||||
**anthropic_messages_optional_request_params,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
|
|||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
from .streaming_iterator import AnthropicResponsesStreamWrapper
|
||||
|
|
@ -34,6 +35,15 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str,
|
|||
return extra_kwargs or {}
|
||||
|
||||
|
||||
def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool:
|
||||
provider: Final = (
|
||||
custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1]
|
||||
)
|
||||
provider_model: Final = local_model_name(model, provider)
|
||||
responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model)
|
||||
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
|
||||
|
||||
|
||||
def _build_responses_kwargs(
|
||||
*,
|
||||
max_tokens: int,
|
||||
|
|
@ -85,8 +95,13 @@ def _build_responses_kwargs(
|
|||
request_data["output_format"] = output_format
|
||||
|
||||
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
|
||||
responses_kwargs: Final = _ADAPTER.translate_request(anthropic_request)
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
responses_kwargs: Final = _ADAPTER.translate_request(
|
||||
anthropic_request,
|
||||
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
|
||||
model, forwarded_kwargs.get("custom_llm_provider")
|
||||
),
|
||||
)
|
||||
|
||||
# Normalize reasoning effort based on model capabilities
|
||||
# (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported)
|
||||
|
|
@ -111,7 +126,7 @@ def _build_responses_kwargs(
|
|||
responses_kwargs["stream"] = True
|
||||
|
||||
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
|
||||
excluded: Final = {"anthropic_messages"}
|
||||
excluded: Final = frozenset(("anthropic_messages",))
|
||||
for key, value in forwarded_kwargs.items():
|
||||
if key == "litellm_logging_obj" and value is not None:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -132,6 +147,14 @@ def _build_responses_kwargs(
|
|||
if explicit_prompt_cache_key is not None:
|
||||
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
|
||||
|
||||
deployment_include: Final = forwarded_kwargs.get("include")
|
||||
bridge_include: Final = responses_kwargs.get("include")
|
||||
if isinstance(deployment_include, list) and isinstance(bridge_include, list):
|
||||
responses_kwargs["include"] = [
|
||||
*bridge_include,
|
||||
*(item for item in deployment_include if item not in bridge_include),
|
||||
]
|
||||
|
||||
return responses_kwargs
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,13 +9,19 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
|
||||
refusal_stop_details,
|
||||
responses_output_refusal_text,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
|
||||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
from .transformation import (
|
||||
REASONING_SUMMARY_PART_SEPARATOR,
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
|
|
@ -29,9 +35,10 @@ class AnthropicResponsesStreamWrapper:
|
|||
response.created -> message_start
|
||||
response.output_item.added -> content_block_start (if message/function_call)
|
||||
response.output_text.delta -> content_block_delta (text_delta)
|
||||
response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator)
|
||||
response.reasoning_summary_text.delta -> content_block_delta (thinking_delta)
|
||||
response.function_call_arguments.delta -> content_block_delta (input_json_delta)
|
||||
response.output_item.done -> content_block_stop
|
||||
response.output_item.done -> content_block_delta (signature_delta) + content_block_stop
|
||||
response.completed -> message_delta + message_stop
|
||||
"""
|
||||
|
||||
|
|
@ -94,6 +101,38 @@ class AnthropicResponsesStreamWrapper:
|
|||
)
|
||||
return block_idx
|
||||
|
||||
@staticmethod
|
||||
def _field(source: object, name: str) -> object:
|
||||
return source.get(name) if isinstance(source, dict) else getattr(source, name, None)
|
||||
|
||||
def _close_reasoning_item(self, item: object, item_id: str | None) -> None:
|
||||
block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
|
||||
encrypted_content: Final = self._field(item, "encrypted_content")
|
||||
signature: Final = (
|
||||
encrypted_reasoning_signature(encrypted_content)
|
||||
if isinstance(encrypted_content, str) and encrypted_content
|
||||
else None
|
||||
)
|
||||
if block_idx < 0 and signature is None:
|
||||
return
|
||||
if block_idx < 0:
|
||||
redacted_idx: Final = self._open_block(
|
||||
item_id,
|
||||
{"type": "redacted_thinking", "data": signature}, # mutable-ok: API message payload
|
||||
)
|
||||
stop: Final = {"type": "content_block_stop", "index": redacted_idx} # mutable-ok: API message payload
|
||||
self._chunk_queue.append(stop)
|
||||
return
|
||||
if signature is not None:
|
||||
self._chunk_queue.append(
|
||||
{ # mutable-ok: API message payload
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "signature_delta", "signature": signature}, # mutable-ok: API message payload
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) # mutable-ok: API message payload
|
||||
|
||||
def _process_event(self, event: object) -> None:
|
||||
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
|
||||
event_type = getattr(event, "type", None)
|
||||
|
|
@ -175,6 +214,26 @@ class AnthropicResponsesStreamWrapper:
|
|||
)
|
||||
return
|
||||
|
||||
if event_type == "response.reasoning_summary_part.added":
|
||||
part_item_id: Final = self._field(event, "item_id")
|
||||
summary_index: Final = self._field(event, "summary_index")
|
||||
part_block_idx: Final = (
|
||||
self._item_id_to_block_index.get(part_item_id, -1) if isinstance(part_item_id, str) else -1
|
||||
)
|
||||
if part_block_idx < 0 or not isinstance(summary_index, int) or summary_index == 0:
|
||||
return
|
||||
self._chunk_queue.append(
|
||||
{ # mutable-ok: API message payload
|
||||
"type": "content_block_delta",
|
||||
"index": part_block_idx,
|
||||
"delta": { # mutable-ok: API message payload
|
||||
"type": "thinking_delta",
|
||||
"thinking": REASONING_SUMMARY_PART_SEPARATOR,
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
# ---- reasoning summary text delta ----
|
||||
if event_type == "response.reasoning_summary_text.delta":
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
|
|
@ -220,6 +279,9 @@ class AnthropicResponsesStreamWrapper:
|
|||
item_id = (
|
||||
getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None
|
||||
)
|
||||
if self._field(item, "type") == "reasoning":
|
||||
self._close_reasoning_item(item, item_id)
|
||||
return
|
||||
block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
|
||||
if block_idx < 0:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ from typing import Any, Final, cast
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
responses_reasoning_item_from_thinking_blocks,
|
||||
encrypted_reasoning_signature,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
|
|
@ -33,6 +34,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicFinishReason,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesToolChoice,
|
||||
AnthropicResponseContentBlockRedactedThinking,
|
||||
AnthropicResponseContentBlockText,
|
||||
AnthropicResponseContentBlockThinking,
|
||||
AnthropicResponseContentBlockToolUse,
|
||||
|
|
@ -43,11 +45,13 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
|
|||
AnthropicUsage,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionThinkingBlock,
|
||||
ResponseAPIUsage,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
|
||||
REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
|
||||
RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
|
||||
|
||||
|
||||
class LiteLLMAnthropicToResponsesAPIAdapter:
|
||||
"""
|
||||
|
|
@ -163,49 +167,55 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
return str(getattr(part, "text", None) or "")
|
||||
|
||||
@classmethod
|
||||
def _thinking_blocks_from_reasoning_item(
|
||||
def _thinking_block_from_reasoning_item(
|
||||
cls,
|
||||
summary: Iterable[object],
|
||||
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
|
||||
"""Anthropic thinking blocks for one Responses reasoning item.
|
||||
encrypted_content: object,
|
||||
) -> dict[str, Any] | None: # mutable-ok: API message payload
|
||||
"""The one Anthropic block for a Responses reasoning item.
|
||||
|
||||
The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
|
||||
value would be replayed as a real one and rejected by every backend that verifies it.
|
||||
The item's encrypted reasoning rides the block's opaque field (`signature`, or
|
||||
`data` when there is no summary text) so the client echoes it back and the next
|
||||
turn replays the very item OpenAI produced; without it the signature stays empty,
|
||||
since only Anthropic can sign a thinking block.
|
||||
"""
|
||||
return tuple(
|
||||
AnthropicResponseContentBlockThinking(
|
||||
type="thinking",
|
||||
thinking=text,
|
||||
signature=None,
|
||||
).model_dump()
|
||||
for part in summary
|
||||
if (text := cls._summary_part_text(part))
|
||||
text: Final = REASONING_SUMMARY_PART_SEPARATOR.join(
|
||||
part_text for part in summary if (part_text := cls._summary_part_text(part))
|
||||
)
|
||||
if not isinstance(encrypted_content, str) or not encrypted_content:
|
||||
if not text:
|
||||
return None
|
||||
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump()
|
||||
signature: Final = encrypted_reasoning_signature(encrypted_content)
|
||||
if not text:
|
||||
return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump()
|
||||
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=signature).model_dump()
|
||||
|
||||
@staticmethod
|
||||
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
|
||||
"""Group a run of consecutive thinking blocks together; keep every other block alone."""
|
||||
index, block = indexed_block
|
||||
return "thinking" if block.get("type") == "thinking" else f"block:{index}"
|
||||
return "thinking" if block.get("type") in ("thinking", "redacted_thinking") else f"block:{index}"
|
||||
|
||||
@classmethod
|
||||
def _assistant_group_to_input_item(
|
||||
def _assistant_group_to_input_items(
|
||||
cls, group: tuple[Mapping[str, object], ...]
|
||||
) -> dict[str, Any] | None: # mutable-ok: API message payload
|
||||
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
|
||||
first: Final = group[0]
|
||||
btype: Final = first.get("type")
|
||||
if btype == "thinking":
|
||||
blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload
|
||||
reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
|
||||
return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload
|
||||
if btype in ("thinking", "redacted_thinking"):
|
||||
replayed: Final = responses_reasoning_items_from_thinking_blocks(group)
|
||||
return tuple(dict(item) for item in replayed) # mutable-ok: API message payload
|
||||
if btype == "tool_use":
|
||||
return { # mutable-ok: API message payload
|
||||
"type": "function_call",
|
||||
"call_id": first.get("id", ""),
|
||||
"name": first.get("name", ""),
|
||||
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
|
||||
}
|
||||
return None
|
||||
return (
|
||||
{ # mutable-ok: API message payload
|
||||
"type": "function_call",
|
||||
"call_id": first.get("id", ""),
|
||||
"name": first.get("name", ""),
|
||||
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
|
||||
},
|
||||
)
|
||||
return ()
|
||||
|
||||
def translate_messages_to_responses_input(
|
||||
self,
|
||||
|
|
@ -362,7 +372,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
input_items.extend(
|
||||
item
|
||||
for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
|
||||
if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
|
||||
for item in self._assistant_group_to_input_items(tuple(block for _, block in group))
|
||||
)
|
||||
asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload
|
||||
{"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload
|
||||
|
|
@ -495,10 +505,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
def translate_request(
|
||||
self,
|
||||
anthropic_request: AnthropicMessagesRequest,
|
||||
include_encrypted_reasoning: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Translate a full Anthropic /v1/messages request dict to
|
||||
litellm.responses() / litellm.aresponses() kwargs.
|
||||
|
||||
``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content``
|
||||
on every call, so a reasoning model's items can be replayed intact next turn even
|
||||
when the client sent no ``thinking`` block; pass False for a provider whose
|
||||
Responses API rejects ``include``.
|
||||
"""
|
||||
model: Final[str] = anthropic_request["model"]
|
||||
messages_list: Final = cast(
|
||||
|
|
@ -528,6 +544,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
"model": model,
|
||||
"input": input_items,
|
||||
}
|
||||
if include_encrypted_reasoning:
|
||||
responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: API request payload
|
||||
|
||||
if system and not developer_parts:
|
||||
if isinstance(system, str):
|
||||
|
|
@ -634,7 +652,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
|
||||
for item in response.output:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
|
||||
reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content)
|
||||
if reasoning_block is not None:
|
||||
content.append(reasoning_block)
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for part in item.content:
|
||||
|
|
@ -684,11 +704,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
).model_dump()
|
||||
)
|
||||
elif item_type == "reasoning":
|
||||
content.extend(
|
||||
self._thinking_blocks_from_reasoning_item(
|
||||
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
|
||||
)
|
||||
reasoning_block = self._thinking_block_from_reasoning_item(
|
||||
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
|
||||
item.get("encrypted_content"),
|
||||
)
|
||||
if reasoning_block is not None:
|
||||
content.append(reasoning_block)
|
||||
elif item_type == "function_call":
|
||||
try:
|
||||
input_data = json.loads(item.get("arguments", "{}"))
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ Safe to enable globally:
|
|||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Final, Optional, Protocol, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -48,6 +48,10 @@ from litellm.exceptions import (
|
|||
ServiceUnavailableError,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_content_of_block,
|
||||
strip_encrypted_reasoning_from_messages,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.cooldown_cache import CooldownCacheValue
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -138,15 +142,48 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
# If no encoded ID, check if encrypted_content itself is wrapped
|
||||
encrypted_content = item.get("encrypted_content")
|
||||
if encrypted_content and isinstance(encrypted_content, str):
|
||||
(
|
||||
model_id,
|
||||
_,
|
||||
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
|
||||
model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content)
|
||||
if model_id:
|
||||
return model_id
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]:
|
||||
if not isinstance(messages, list):
|
||||
return iter(())
|
||||
return (
|
||||
cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
|
||||
for message in cast(list[object], messages) # cast-ok: narrowed by isinstance
|
||||
if isinstance(message, Mapping)
|
||||
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
|
||||
if isinstance(content, list)
|
||||
for block in cast(list[object], content) # cast-ok: narrowed by isinstance
|
||||
if isinstance(block, Mapping)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _model_id_from_wrapped_encrypted_content(encrypted_content: str) -> str | None:
|
||||
model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
|
||||
return model_id or None
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id_from_anthropic_messages(messages: object) -> str | None:
|
||||
return next(
|
||||
(
|
||||
model_id
|
||||
for block in EncryptedContentAffinityCheck._anthropic_content_blocks(messages)
|
||||
if (encrypted_content := encrypted_content_of_block(block)) is not None
|
||||
if (
|
||||
model_id := EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(
|
||||
encrypted_content
|
||||
)
|
||||
)
|
||||
is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
|
||||
for deployment in healthy_deployments:
|
||||
|
|
@ -240,8 +277,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
parent_otel_span: Span | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
If the request ``input`` contains litellm-encoded item IDs, decode the
|
||||
embedded ``model_id`` and pin the request to that deployment. Raises
|
||||
If the request ``input`` contains litellm-encoded item IDs, or its Anthropic
|
||||
``messages`` replay a bridge-tagged thinking block, decode the embedded
|
||||
``model_id`` and pin the request to that deployment. Raises
|
||||
``RateLimitError`` / ``ServiceUnavailableError`` when the originating
|
||||
deployment is a member of the routed model group but currently unavailable
|
||||
and no encryption-boundary peer exists, rather than dispatching a doomed
|
||||
|
|
@ -270,12 +308,15 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True
|
||||
|
||||
request_input: Final = request_kwargs.get("input")
|
||||
model_id: Final = self._extract_model_id_from_input(request_input)
|
||||
anthropic_messages: Final = messages or request_kwargs.get("messages")
|
||||
model_id: Final = self._extract_model_id_from_input(
|
||||
request_input
|
||||
) or self._extract_model_id_from_anthropic_messages(anthropic_messages)
|
||||
if not model_id:
|
||||
return typed_healthy_deployments
|
||||
|
||||
verbose_router_logger.debug(
|
||||
"EncryptedContentAffinityCheck: decoded model_id=%s from input item IDs",
|
||||
"EncryptedContentAffinityCheck: decoded model_id=%s from the request's encrypted content markers",
|
||||
model_id,
|
||||
)
|
||||
|
||||
|
|
@ -327,6 +368,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
model,
|
||||
)
|
||||
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input)
|
||||
strip_encrypted_reasoning_from_messages(anthropic_messages)
|
||||
return typed_healthy_deployments
|
||||
|
||||
# The origin is a member of the routed group but currently unavailable (cooled down); fail fast
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
|
|
@ -7,14 +8,20 @@ import pytest
|
|||
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
ENCRYPTED_REASONING_SIGNATURE_PREFIX,
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
add_system_prompt_to_messages,
|
||||
encrypted_content_from_signature,
|
||||
encrypted_reasoning_signature,
|
||||
get_file_ids_from_messages,
|
||||
get_format_from_file_id,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
split_concatenated_json_objects,
|
||||
strip_encrypted_reasoning_from_messages,
|
||||
update_messages_with_model_file_ids,
|
||||
)
|
||||
|
||||
|
|
@ -1554,3 +1561,117 @@ class TestRequestContainsImageContent:
|
|||
for _ in range(50):
|
||||
nested = {"type": "tool_result", "content": [nested]}
|
||||
assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False
|
||||
|
||||
|
||||
class TestEncryptedReasoningReplay:
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/40288."""
|
||||
|
||||
def test_signature_round_trips_the_encrypted_content(self):
|
||||
assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]
|
||||
)
|
||||
def test_anything_else_is_not_encrypted_content(self, signature):
|
||||
assert encrypted_content_from_signature(signature) is None
|
||||
|
||||
def test_encrypted_thinking_block_replays_its_own_item(self):
|
||||
items = responses_reasoning_items_from_thinking_blocks(
|
||||
[{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}]
|
||||
)
|
||||
assert items == (
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "Plan."}],
|
||||
"encrypted_content": "gAAAA_1",
|
||||
},
|
||||
)
|
||||
|
||||
def test_encrypted_redacted_block_replays_with_an_empty_summary(self):
|
||||
items = responses_reasoning_items_from_thinking_blocks(
|
||||
[{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}]
|
||||
)
|
||||
assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},)
|
||||
|
||||
def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self):
|
||||
items = responses_reasoning_items_from_thinking_blocks(
|
||||
[
|
||||
{"type": "thinking", "thinking": "A.", "signature": None},
|
||||
{"type": "thinking", "thinking": "B.", "signature": ""},
|
||||
{"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")},
|
||||
{"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"},
|
||||
{"type": "thinking", "thinking": "D."},
|
||||
]
|
||||
)
|
||||
assert items == (
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}],
|
||||
},
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"},
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]},
|
||||
)
|
||||
assert all("id" not in item for item in items)
|
||||
|
||||
def test_blocks_without_text_or_encrypted_content_produce_nothing(self):
|
||||
assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == ()
|
||||
assert responses_reasoning_items_from_thinking_blocks([]) == ()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block", "expected"),
|
||||
[
|
||||
({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True),
|
||||
({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True),
|
||||
({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True),
|
||||
({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True),
|
||||
({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False),
|
||||
({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False),
|
||||
({"type": "text", "text": encrypted_reasoning_signature("g")}, False),
|
||||
("not a block", False),
|
||||
],
|
||||
)
|
||||
def test_is_encrypted_reasoning_block(self, block, expected):
|
||||
assert is_encrypted_reasoning_block(block) is expected
|
||||
|
||||
def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self):
|
||||
assistant_content = [
|
||||
{"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"},
|
||||
{"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")},
|
||||
{"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")},
|
||||
{"type": "text", "text": "answer"},
|
||||
]
|
||||
messages = [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": assistant_content},
|
||||
{"role": "user", "content": [{"type": "text", "text": "follow-up"}]},
|
||||
]
|
||||
|
||||
strip_encrypted_reasoning_from_messages(messages)
|
||||
|
||||
assert messages[1]["content"] is assistant_content
|
||||
assert assistant_content == [
|
||||
{"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"},
|
||||
{"type": "text", "text": "answer"},
|
||||
]
|
||||
assert all(block["signature"] for block in assistant_content if block["type"] == "thinking")
|
||||
assert messages[0] == {"role": "user", "content": "question"}
|
||||
assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"messages",
|
||||
[
|
||||
"not a list",
|
||||
None,
|
||||
[{"role": "user", "content": None}],
|
||||
[{"role": "user", "content": "plain string"}],
|
||||
["not a message"],
|
||||
[{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}],
|
||||
],
|
||||
)
|
||||
def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages):
|
||||
before = copy.deepcopy(messages)
|
||||
|
||||
strip_encrypted_reasoning_from_messages(messages)
|
||||
|
||||
assert messages == before
|
||||
|
|
|
|||
|
|
@ -191,8 +191,16 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls():
|
|||
{"type": "thinking", "thinking": "oss reasoning", "signature": None},
|
||||
{"type": "thinking", "thinking": "oss reasoning", "signature": ""},
|
||||
{"type": "thinking", "thinking": "oss reasoning"},
|
||||
{"type": "thinking", "thinking": "openai reasoning", "signature": "litellm_encrypted_reasoning:gAAAA"},
|
||||
{"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"},
|
||||
],
|
||||
ids=[
|
||||
"null_signature",
|
||||
"empty_signature",
|
||||
"missing_signature",
|
||||
"encrypted_reasoning_signature",
|
||||
"encrypted_reasoning_redacted_data",
|
||||
],
|
||||
ids=["null_signature", "empty_signature", "missing_signature"],
|
||||
)
|
||||
def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block):
|
||||
"""Open-source reasoning models (DeepSeek-R1, Qwen, etc.) emit thinking blocks
|
||||
|
|
@ -219,7 +227,7 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block):
|
|||
assistant = next(m for m in result if m["role"] == "assistant")
|
||||
content = assistant["content"]
|
||||
assert all(
|
||||
block.get("type") != "thinking" for block in content
|
||||
block.get("type") not in ("thinking", "redacted_thinking") for block in content
|
||||
), f"unsignable thinking block must be dropped, got {content!r}"
|
||||
assert any(
|
||||
block.get("type") == "text" and block.get("text") == "2+2 equals 4."
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import litellm
|
|||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
|
|
@ -423,6 +424,43 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks():
|
|||
assert result[1]["tool_calls"][0]["id"] == "toolu_01234"
|
||||
|
||||
|
||||
def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning_blocks():
|
||||
"""A session that moves from an OpenAI reasoning model to a chat provider replays reasoning only OpenAI can read.
|
||||
|
||||
Gemini rejects the whole request when such a block reaches it as a thought_signature, so the
|
||||
adapter drops those blocks and keeps the provider-signed ones.
|
||||
"""
|
||||
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[{"type": "text", "text": "Who drinks water?"}],
|
||||
),
|
||||
AnthopicMessagesAssistantMessageParam(
|
||||
role="assistant",
|
||||
content=[
|
||||
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
|
||||
{"type": "text", "text": "The Norwegian."},
|
||||
],
|
||||
),
|
||||
AnthopicMessagesAssistantMessageParam(
|
||||
role="assistant",
|
||||
content=[
|
||||
{"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_signed"},
|
||||
{"type": "text", "text": "Still the Norwegian."},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages)
|
||||
|
||||
assert [m["role"] for m in result] == ["user", "assistant", "assistant"]
|
||||
assert not result[1].get("thinking_blocks")
|
||||
assert result[1]["content"] == "The Norwegian."
|
||||
assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"]
|
||||
|
||||
|
||||
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
|
||||
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
|
||||
def _transform(messages):
|
||||
return AnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="claude-sonnet-4-5",
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params={"max_tokens": 1024},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_replayed_from_the_responses_bridge_never_reaches_anthropic():
|
||||
"""Claude Code resumed on a Claude model echoes the thinking blocks a gpt turn produced."""
|
||||
messages = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
|
||||
{"type": "text", "text": "The answer."},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "And the next one?"},
|
||||
]
|
||||
request = _transform(messages)
|
||||
assert request["messages"][1]["content"] == [{"type": "text", "text": "The answer."}]
|
||||
assert len(messages[1]["content"]) == 3
|
||||
|
||||
|
||||
def test_anthropic_signed_thinking_blocks_are_forwarded_untouched():
|
||||
messages = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
|
||||
{"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"},
|
||||
{"type": "text", "text": "The answer."},
|
||||
],
|
||||
},
|
||||
]
|
||||
assert _transform(messages)["messages"] == messages
|
||||
|
|
@ -67,6 +67,39 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
|
|||
assert responses_kwargs["prompt_cache_key"] == "explicit-key"
|
||||
|
||||
|
||||
def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
extra_kwargs={"custom_llm_provider": "openai"},
|
||||
)
|
||||
assert responses_kwargs["include"] == ["reasoning.encrypted_content"]
|
||||
assert "reasoning" not in responses_kwargs
|
||||
|
||||
|
||||
def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="perplexity/sonar",
|
||||
thinking={"type": "enabled", "budget_tokens": 4096},
|
||||
extra_kwargs={"custom_llm_provider": "perplexity"},
|
||||
)
|
||||
assert "include" not in responses_kwargs
|
||||
assert "reasoning" in responses_kwargs
|
||||
|
||||
|
||||
def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
extra_kwargs={"custom_llm_provider": "openai", "include": ["file_search_call.results"]},
|
||||
)
|
||||
assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"]
|
||||
|
||||
|
||||
def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ from types import SimpleNamespace
|
|||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import (
|
||||
AnthropicResponsesStreamWrapper,
|
||||
)
|
||||
|
|
@ -114,7 +117,7 @@ class TestReasoningItemWithoutSummaryText:
|
|||
"""
|
||||
|
||||
@staticmethod
|
||||
def _gpt_turn(reasoning_summary_deltas: list) -> list:
|
||||
def _gpt_turn(reasoning_summary_deltas: list, encrypted_content: str | None = None) -> list:
|
||||
return [
|
||||
{"type": "response.created"},
|
||||
{"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
|
|
@ -122,7 +125,10 @@ class TestReasoningItemWithoutSummaryText:
|
|||
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta}
|
||||
for delta in reasoning_summary_deltas
|
||||
),
|
||||
{"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted_content},
|
||||
},
|
||||
{"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}},
|
||||
{"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"},
|
||||
{"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}},
|
||||
|
|
@ -171,6 +177,70 @@ class TestReasoningItemWithoutSummaryText:
|
|||
assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"]
|
||||
|
||||
|
||||
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
|
||||
|
||||
|
||||
class TestEncryptedReasoningIsStreamedForReplay:
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/40288.
|
||||
|
||||
The client echoes a thinking block's signature (or a redacted block's data) back on the
|
||||
next turn, so the item's ``encrypted_content`` has to reach it through one of those.
|
||||
"""
|
||||
|
||||
def test_encrypted_content_is_streamed_as_the_signature_before_the_block_closes(self):
|
||||
chunks = _drain_async(
|
||||
TestReasoningItemWithoutSummaryText._gpt_turn(
|
||||
reasoning_summary_deltas=["Weighing options"], encrypted_content=_ENCRYPTED_REASONING
|
||||
)
|
||||
)
|
||||
|
||||
assert [(c["type"], c.get("index"), c.get("delta", {}).get("type")) for c in chunks[1:5]] == [
|
||||
("content_block_start", 0, None),
|
||||
("content_block_delta", 0, "thinking_delta"),
|
||||
("content_block_delta", 0, "signature_delta"),
|
||||
("content_block_stop", 0, None),
|
||||
]
|
||||
assert chunks[3]["delta"]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING)
|
||||
|
||||
def test_reasoning_without_summary_streams_a_redacted_thinking_block(self):
|
||||
chunks = _drain_async(
|
||||
TestReasoningItemWithoutSummaryText._gpt_turn(
|
||||
reasoning_summary_deltas=[], encrypted_content=_ENCRYPTED_REASONING
|
||||
)
|
||||
)
|
||||
|
||||
assert [(c["type"], c.get("index")) for c in chunks[1:]] == [
|
||||
("content_block_start", 0),
|
||||
("content_block_stop", 0),
|
||||
("content_block_start", 1),
|
||||
("content_block_delta", 1),
|
||||
("content_block_stop", 1),
|
||||
]
|
||||
assert chunks[1]["content_block"] == {
|
||||
"type": "redacted_thinking",
|
||||
"data": encrypted_reasoning_signature(_ENCRYPTED_REASONING),
|
||||
}
|
||||
|
||||
def test_summary_parts_are_separated_inside_the_one_thinking_block(self):
|
||||
"""Two summary parts read as two paragraphs, not as one run-on sentence."""
|
||||
events = [
|
||||
{"type": "response.created"},
|
||||
{"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
{"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 0},
|
||||
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "First."},
|
||||
{"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 1},
|
||||
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "Second."},
|
||||
{"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
]
|
||||
chunks = _process_all(events)
|
||||
|
||||
thinking = "".join(
|
||||
c["delta"]["thinking"] for c in chunks if c.get("delta", {}).get("type") == "thinking_delta"
|
||||
)
|
||||
assert thinking == "First.\n\nSecond."
|
||||
assert [c["type"] for c in chunks].count("content_block_start") == 1
|
||||
|
||||
|
||||
class TestToolUseBlockClosedExactlyOnce:
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/37273.
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
|
|
@ -566,6 +567,66 @@ class TestTranslateMessagesToResponsesInput:
|
|||
result = _translate_messages(messages)
|
||||
assert "id" not in result[0]
|
||||
|
||||
def test_thinking_block_with_encrypted_signature_replays_the_encrypted_content(self):
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/40288 (inbound fault site)."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "Private reasoning.",
|
||||
"signature": encrypted_reasoning_signature("gAAAA_turn_one"),
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
result = _translate_messages(messages)
|
||||
assert result == [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "Private reasoning."}],
|
||||
"encrypted_content": "gAAAA_turn_one",
|
||||
}
|
||||
]
|
||||
|
||||
def test_redacted_thinking_with_encrypted_data_replays_the_encrypted_content(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_turn_one")}],
|
||||
}
|
||||
]
|
||||
result = _translate_messages(messages)
|
||||
assert result == [{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_turn_one"}]
|
||||
|
||||
def test_each_encrypted_thinking_block_stays_its_own_reasoning_item(self):
|
||||
"""Two upstream items must not be merged into one, or the encrypted content of one is lost."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "First.", "signature": encrypted_reasoning_signature("gAAAA_1")},
|
||||
{"type": "thinking", "thinking": "Second.", "signature": encrypted_reasoning_signature("gAAAA_2")},
|
||||
],
|
||||
}
|
||||
]
|
||||
result = _translate_messages(messages)
|
||||
assert [item["encrypted_content"] for item in result] == ["gAAAA_1", "gAAAA_2"]
|
||||
|
||||
def test_anthropic_signed_thinking_block_replays_as_a_summary_only_item(self):
|
||||
"""A real Anthropic signature is opaque here, so it never masquerades as encrypted content."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "ErcBCkgIValid"}],
|
||||
}
|
||||
]
|
||||
result = _translate_messages(messages)
|
||||
assert result == [
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]}
|
||||
]
|
||||
|
||||
def test_consecutive_thinking_blocks_become_one_reasoning_item(self):
|
||||
"""Summary parts of one upstream reasoning item are regrouped into that item."""
|
||||
messages = [
|
||||
|
|
@ -1102,6 +1163,23 @@ class TestTranslateRequestBroaderCoverage:
|
|||
kwargs = _ADAPTER.translate_request(req)
|
||||
assert "reasoning" not in kwargs
|
||||
|
||||
def test_thinking_asks_for_the_encrypted_reasoning(self):
|
||||
"""The documented way to get reasoning that survives store=false is to ask for it."""
|
||||
req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000})
|
||||
kwargs = _ADAPTER.translate_request(req)
|
||||
assert kwargs["include"] == ["reasoning.encrypted_content"]
|
||||
|
||||
def test_encrypted_reasoning_is_asked_for_without_a_thinking_block(self):
|
||||
"""A reasoning model reasons whether or not the client sent `thinking`, so the replay needs it either way."""
|
||||
kwargs = _ADAPTER.translate_request(_make_request())
|
||||
assert kwargs["include"] == ["reasoning.encrypted_content"]
|
||||
|
||||
def test_encrypted_reasoning_is_not_asked_for_when_the_provider_rejects_include(self):
|
||||
req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000})
|
||||
kwargs = _ADAPTER.translate_request(req, include_encrypted_reasoning=False)
|
||||
assert kwargs["reasoning"] == {"effort": "high"}
|
||||
assert "include" not in kwargs
|
||||
|
||||
def test_metadata_user_id_mapped_to_user(self):
|
||||
req = _make_request(metadata={"user_id": "user-42"})
|
||||
kwargs = _ADAPTER.translate_request(req)
|
||||
|
|
@ -1246,7 +1324,9 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo
|
|||
return item
|
||||
|
||||
|
||||
def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock:
|
||||
def _make_reasoning_item(
|
||||
summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None
|
||||
) -> MagicMock:
|
||||
"""Build a mock ResponseReasoningItem."""
|
||||
from openai.types.responses import ResponseReasoningItem # type: ignore[import]
|
||||
|
||||
|
|
@ -1259,9 +1339,13 @@ def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> Ma
|
|||
item = MagicMock(spec=ResponseReasoningItem)
|
||||
item.id = item_id
|
||||
item.summary = summary_mocks
|
||||
item.encrypted_content = encrypted_content
|
||||
return item
|
||||
|
||||
|
||||
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
|
||||
|
||||
|
||||
class TestTranslateResponse:
|
||||
"""Responses API -> AnthropicMessagesResponse conversion."""
|
||||
|
||||
|
|
@ -1386,7 +1470,81 @@ class TestTranslateResponse:
|
|||
reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123")
|
||||
response = _make_mock_response(output=[reasoning])
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert [block["signature"] for block in result["content"]] == [None, None]
|
||||
assert [block["signature"] for block in result["content"]] == [None]
|
||||
assert "rs_abc123" not in json.dumps(result["content"])
|
||||
|
||||
def test_summary_parts_join_into_one_thinking_block(self):
|
||||
"""One reasoning item is one block, so its signature is echoed back exactly once."""
|
||||
reasoning = _make_reasoning_item(["Part one.", "Part two."])
|
||||
response = _make_mock_response(output=[reasoning])
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert [block["thinking"] for block in result["content"]] == ["Part one.\n\nPart two."]
|
||||
|
||||
def test_encrypted_content_rides_the_thinking_signature(self):
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/40288 (outbound fault site)."""
|
||||
reasoning = _make_reasoning_item(["Part one."], item_id="rs_abc123", encrypted_content=_ENCRYPTED_REASONING)
|
||||
response = _make_mock_response(output=[reasoning])
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert result["content"] == [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "Part one.",
|
||||
"signature": encrypted_reasoning_signature(_ENCRYPTED_REASONING),
|
||||
}
|
||||
]
|
||||
|
||||
def test_reasoning_without_summary_becomes_redacted_thinking(self):
|
||||
"""With summaries off the encrypted reasoning still has to reach the client to be replayed."""
|
||||
reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING)
|
||||
response = _make_mock_response(output=[reasoning])
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert result["content"] == [
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING)}
|
||||
]
|
||||
|
||||
def test_dict_reasoning_item_carries_its_encrypted_content(self):
|
||||
response = _make_mock_response(
|
||||
output=[
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_dict_1",
|
||||
"encrypted_content": _ENCRYPTED_REASONING,
|
||||
"summary": [{"type": "summary_text", "text": "Weighing the options."}],
|
||||
}
|
||||
]
|
||||
)
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert result["content"][0]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING)
|
||||
|
||||
def test_reasoning_item_round_trip_is_byte_stable(self):
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/40288.
|
||||
|
||||
The reasoning item the next turn replays must be the one OpenAI produced, with its
|
||||
encrypted reasoning intact, and identical on every later turn so the prompt cache
|
||||
prefix keeps matching.
|
||||
"""
|
||||
reasoning = _make_reasoning_item(["Part one.", "Part two."], encrypted_content=_ENCRYPTED_REASONING)
|
||||
turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning]))
|
||||
history = [{"role": "assistant", "content": turn["content"]}]
|
||||
|
||||
replayed_items = [_translate_messages(history) for _ in range(2)]
|
||||
|
||||
assert replayed_items[0] == replayed_items[1]
|
||||
assert replayed_items[0] == [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "Part one.\n\nPart two."}],
|
||||
"encrypted_content": _ENCRYPTED_REASONING,
|
||||
}
|
||||
]
|
||||
|
||||
def test_redacted_reasoning_round_trip_replays_the_encrypted_content(self):
|
||||
reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING)
|
||||
turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning]))
|
||||
|
||||
replayed = _translate_messages([{"role": "assistant", "content": turn["content"]}])
|
||||
|
||||
assert replayed == [{"type": "reasoning", "summary": [], "encrypted_content": _ENCRYPTED_REASONING}]
|
||||
|
||||
def test_dict_reasoning_item_becomes_thinking_block(self):
|
||||
"""A reasoning item arriving as a plain dict is kept, not dropped."""
|
||||
|
|
@ -1402,14 +1560,26 @@ class TestTranslateResponse:
|
|||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}]
|
||||
|
||||
def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self):
|
||||
@pytest.mark.parametrize(
|
||||
("summaries", "encrypted_content"),
|
||||
[
|
||||
(["Part one."], None),
|
||||
(["Part one."], _ENCRYPTED_REASONING),
|
||||
([], _ENCRYPTED_REASONING),
|
||||
],
|
||||
ids=["unsigned_thinking", "encrypted_thinking", "encrypted_redacted_thinking"],
|
||||
)
|
||||
def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self, summaries, encrypted_content):
|
||||
"""Replaying this turn to an Anthropic model must not send a signature it cannot verify."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
_drop_unsignable_thinking_blocks,
|
||||
)
|
||||
|
||||
response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")])
|
||||
response = _make_mock_response(
|
||||
output=[_make_reasoning_item(summaries, item_id="rs_abc123", encrypted_content=encrypted_content)]
|
||||
)
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert len(result["content"]) == 1
|
||||
assert _drop_unsignable_thinking_blocks(result["content"]) == []
|
||||
|
||||
def test_usage_mapped_correctly(self):
|
||||
|
|
|
|||
|
|
@ -42,12 +42,15 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789"
|
|||
def test_is_claude_code_one_shot_subagent_request(messages, system, expected):
|
||||
from litellm.llms.anthropic.common_utils import is_claude_code_one_shot_subagent_request
|
||||
|
||||
assert is_claude_code_one_shot_subagent_request(
|
||||
messages=messages,
|
||||
system=system,
|
||||
tools=None,
|
||||
user_agent="claude-cli/2.1.263 (external, cli)",
|
||||
) is expected
|
||||
assert (
|
||||
is_claude_code_one_shot_subagent_request(
|
||||
messages=messages,
|
||||
system=system,
|
||||
tools=None,
|
||||
user_agent="claude-cli/2.1.263 (external, cli)",
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
class TestOptionallyHandleAnthropicOAuth:
|
||||
|
|
@ -1541,6 +1544,71 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
out = strip_empty_content_blocks_from_anthropic_messages(msgs)
|
||||
assert [b["type"] for b in out[0]["content"]] == ["thinking"]
|
||||
|
||||
def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self):
|
||||
"""The /v1/messages handler runs this before dispatch, so the bridge must still see the replay."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_empty_content_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
|
||||
{"type": "text", "text": "The answer."},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert strip_empty_content_blocks_from_anthropic_messages(msgs) == msgs
|
||||
|
||||
def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self):
|
||||
"""A session resumed on an Anthropic model replays reasoning only OpenAI can verify."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_3")},
|
||||
{"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
|
||||
{"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"},
|
||||
{"type": "text", "text": "The answer."},
|
||||
],
|
||||
},
|
||||
]
|
||||
out = strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs)
|
||||
assert [m["role"] for m in out] == ["user", "assistant"]
|
||||
assert [b["type"] for b in out[1]["content"]] == ["thinking", "redacted_thinking", "text"]
|
||||
assert out[1]["content"][0]["signature"] == "EqQBCkYIAxgCIkA_anthropic_signed"
|
||||
assert len(msgs[1]["content"]) == 2
|
||||
assert len(msgs[2]["content"]) == 4
|
||||
|
||||
def test_strip_encrypted_reasoning_leaves_malformed_messages_for_the_provider_to_reject(self):
|
||||
"""A bare string in messages must reach Anthropic as a 400, not die in the stripper as a 500."""
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = ["hi", {"role": "user", "content": "hello"}]
|
||||
assert strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) == msgs
|
||||
|
||||
def test_strip_empty_text_blocks_treats_null_text_as_empty(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_empty_content_blocks_from_anthropic_messages,
|
||||
|
|
|
|||
|
|
@ -1584,6 +1584,89 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen
|
|||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_through_the_bridge():
|
||||
"""
|
||||
Claude Code behind /v1/messages replays the encrypted reasoning the bridge packed
|
||||
into a thinking block's signature (or a redacted block's data). The pin has to be
|
||||
read from those blocks because the bridge builds the Responses `input` only after
|
||||
the router has picked a deployment.
|
||||
"""
|
||||
check = EncryptedContentAffinityCheck()
|
||||
deployments = [
|
||||
{"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}},
|
||||
{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}},
|
||||
]
|
||||
request_kwargs = {"model": "gpt-5.1"}
|
||||
|
||||
pinned = await check.async_filter_deployments(
|
||||
model="gpt-5.1",
|
||||
healthy_deployments=deployments,
|
||||
messages=_bridge_replayed_anthropic_messages(minted_by="openai-org-b"),
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
assert [d["model_info"]["id"] for d in pinned] == ["openai-org-b"]
|
||||
assert request_kwargs["_encrypted_content_affinity_pinned"] is True
|
||||
|
||||
|
||||
def _bridge_replayed_anthropic_messages(minted_by: str) -> list:
|
||||
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by)
|
||||
return [
|
||||
{"role": "user", "content": "Solve the zebra puzzle"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
|
||||
{"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"},
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "The bridge packed this one",
|
||||
"signature": f"litellm_encrypted_reasoning:{wrapped}",
|
||||
},
|
||||
{"type": "text", "text": "The zebra owner lives in the green house."},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "And who drinks water?"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group():
|
||||
"""
|
||||
The /v1/messages twin of the tier-change case: the routed group holds no deployment
|
||||
of the org that minted the reasoning, so the bridge-tagged blocks are dropped whole
|
||||
and the request dispatches to the routed pool. No unsigned thinking block may be left
|
||||
behind: Anthropic and Bedrock reject a thinking block with a missing signature the
|
||||
same way they reject a foreign one.
|
||||
"""
|
||||
originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier")
|
||||
mock_router = _make_router_mock_with_cooldown(
|
||||
originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"]
|
||||
)
|
||||
check = EncryptedContentAffinityCheck(router=mock_router)
|
||||
routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}]
|
||||
messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a")
|
||||
assistant_content = messages[1]["content"]
|
||||
request_kwargs = {"model": "gpt-5.1"}
|
||||
|
||||
result = await check.async_filter_deployments(
|
||||
model="gpt-simple-tier",
|
||||
healthy_deployments=routed_pool,
|
||||
messages=messages,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
assert result is routed_pool
|
||||
assert "_encrypted_content_affinity_pinned" not in request_kwargs
|
||||
assert messages[1]["content"] is assistant_content
|
||||
assert assistant_content == [
|
||||
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
|
||||
{"type": "text", "text": "The zebra owner lives in the green house."},
|
||||
]
|
||||
assert all(block["signature"] for block in assistant_content if block["type"] == "thinking")
|
||||
|
||||
|
||||
class TestStripEncryptedReasoningFromInput:
|
||||
def test_keeps_summary_and_drops_encrypted_content_and_id(self):
|
||||
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue