mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(anthropic): close /v1/messages thinking blocks with a signature and keep bridged output (#52116)
* fix(anthropic): close /v1/messages thinking blocks with a signature and keep bridged output * fix(anthropic): close rebuilt-stream thinking blocks with a signature_delta * fix(anthropic): drop unsigned thinking before Claude, stream raw reasoning_text, close rebuilt thinking blocks with signature_delta A Claude target on /v1/messages now gets unsigned thinking blocks left out of the request, so a switch from a reasoning_content model makes one call and keeps thinking, except when that would leave the final tool turn without its opening thinking block Raw response.reasoning_text deltas from the Responses bridge stream as thinking, and a stream rebuilt from a full answer or rewritten by a guardrail closes each thinking block with a signature_delta, the real signature or an empty string * fix(anthropic): stream a whole reasoning item as a closed thinking block A Responses stream can send a reasoning item whole in output_item.added and output_item.done with no text deltas, as litellm's own stream does for gpt-oss on OpenRouter. The /v1/messages stream now builds that item's thinking block with the same helper the non-streamed response uses, so streamed and non-streamed answers carry the same reasoning * test(anthropic): mark the new streaming test locals Final * fix(anthropic): drop unsigned thinking before every Claude call, a pending tool turn included Anthropic answers 400 to a thinking block with an empty signature but accepts a tool turn with no thinking, so the strip no longer keeps the block that opens the pending tool turn. The check now looks for "claude" in the model name, which covers Claude behind every messages route and the chat adapter, not only four providers * fix(anthropic): strip unsigned thinking when a deployment's base_model names Claude * fix(anthropic): drop unsigned thinking for an alias whose base_model names Claude on the native messages path The chat adapter already read base_model before leaving unsigned thinking out of a Claude request. The native messages transform and the Rust request transform now read it too, so an aliased Claude deployment gets one call instead of a 400 and a retry * test(anthropic): build the pending tool turn without mutating it in the Rust strip test The Rust test now builds its expected messages from the same helper instead of editing them afterward, and a doc comment that only restated a helper's name is gone --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
fea939843d
commit
ea96ea3db5
29 changed files with 909 additions and 191 deletions
|
|
@ -574,6 +574,11 @@ def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_lin
|
|||
return None
|
||||
|
||||
|
||||
def _mapping_type(value: object) -> object:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
return mapping.get("type") if mapping is not None else None
|
||||
|
||||
|
||||
class AnthropicMessagesHandler(BaseTranslation):
|
||||
"""Process Anthropic messages with guardrails.
|
||||
|
||||
|
|
@ -698,6 +703,17 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
open_index, max_index = self._content_block_state(responses_so_far)
|
||||
new_index: Final = (max_index + 1) if max_index is not None else 0
|
||||
chunks: list[bytes] = []
|
||||
if open_index is not None and self._is_unsigned_thinking_block(responses_so_far, open_index):
|
||||
chunks.append(
|
||||
_sse(
|
||||
"content_block_delta",
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": open_index,
|
||||
"delta": {"type": "signature_delta", "signature": ""},
|
||||
},
|
||||
)
|
||||
)
|
||||
if open_index is not None:
|
||||
chunks.append(_sse("content_block_stop", {"type": "content_block_stop", "index": open_index}))
|
||||
chunks += [
|
||||
|
|
@ -730,6 +746,18 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
]
|
||||
return chunks
|
||||
|
||||
@staticmethod
|
||||
def _is_unsigned_thinking_block(responses_so_far: Sequence[object], index: int) -> bool:
|
||||
events: Final = tuple(
|
||||
event
|
||||
for event in chain.from_iterable(map(AnthropicMessagesHandler._iter_sse_events, responses_so_far))
|
||||
if event.get("index") == index
|
||||
)
|
||||
return any(
|
||||
event.get("type") == "content_block_start" and _mapping_type(event.get("content_block")) == "thinking"
|
||||
for event in events
|
||||
) and not any(_mapping_type(event.get("delta")) == "signature_delta" for event in events)
|
||||
|
||||
@staticmethod
|
||||
def _content_block_state(
|
||||
responses_so_far: Sequence[object],
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, lax_bool
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
get_file_ids_from_messages,
|
||||
|
|
@ -41,6 +41,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
is_encrypted_reasoning_block,
|
||||
is_unsignable_thinking_block,
|
||||
native_redacted_thinking_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
|
||||
|
|
@ -2187,6 +2188,36 @@ def strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
|||
return [m for m in stripped if m is not None]
|
||||
|
||||
|
||||
def _without_unsignable_thinking(
|
||||
message: dict[str, object], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
|
||||
) -> dict[str, object] | None: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
|
||||
fields: Final = as_str_mapping(message)
|
||||
content: Final = as_object_list(fields.get("content")) if fields is not None else None
|
||||
if content is None:
|
||||
return message
|
||||
kept: Final = [block for block in content if not is_unsignable_thinking_block(block)]
|
||||
if len(kept) == len(content):
|
||||
return message
|
||||
return {**message, "content": kept} if kept else None
|
||||
|
||||
|
||||
def strip_unsignable_thinking_blocks_for_claude(
|
||||
messages: list[dict[str, object]], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
|
||||
model: str,
|
||||
base_model: str | None = None,
|
||||
) -> list[dict[str, object]]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
|
||||
"""
|
||||
Drop thinking blocks Claude cannot verify (no signature, or another provider's
|
||||
encrypted reasoning) before /v1/messages sends the request to a Claude model on
|
||||
the native or chat adapter route, since Claude answers 400 to them, a pending
|
||||
tool turn included. Other targets (DeepSeek, MiniMax, GLM) need their unsigned
|
||||
reasoning replayed.
|
||||
"""
|
||||
if not any("claude" in name.lower() for name in (model, base_model or "")):
|
||||
return messages
|
||||
return [m for m in map(_without_unsignable_thinking, messages) if m is not None]
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
||||
data: dict[str, object],
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,10 @@ from litellm.litellm_core_utils.tool_fields import (
|
|||
drops_mcp_servers,
|
||||
tools_without_mcp_servers,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
strip_unsignable_thinking_blocks_for_claude,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
openai_thinking_routes_to_responses,
|
||||
|
|
@ -744,6 +747,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
)
|
||||
|
||||
provider_kwargs: Final = extra_kwargs or {}
|
||||
base_model: Final = provider_kwargs.get("base_model")
|
||||
custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider")
|
||||
target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
|
||||
drop_params: Final = drops_unsupported_params(provider_kwargs.get("drop_params"))
|
||||
|
|
@ -762,7 +766,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
|
||||
request_data: Final[dict[str, object]] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"messages": strip_unsignable_thinking_blocks_for_claude(
|
||||
messages, model, base_model if isinstance(base_model, str) else None
|
||||
),
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
import copy
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
|
|
@ -15,6 +14,7 @@ from typing import (
|
|||
Final,
|
||||
Literal,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast,
|
||||
get_args,
|
||||
runtime_checkable,
|
||||
|
|
@ -42,6 +42,7 @@ from litellm.types.llms.anthropic import (
|
|||
AppliedEdit,
|
||||
CompactionBlock,
|
||||
ContentBlockDelta,
|
||||
ContentThinkingSignatureBlockDelta,
|
||||
ContextManagementResponse,
|
||||
MessageBlockDelta,
|
||||
MessageDelta,
|
||||
|
|
@ -68,6 +69,7 @@ _SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
|
|||
_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
|
||||
_PROMPT_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
|
||||
_PROMPT_TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam])
|
||||
_AnthropicStreamEvent: TypeAlias = dict[str, object] | MessageBlockDelta | ContentBlockDelta
|
||||
|
||||
|
||||
class _UsageDeltaWithIterations(UsageDelta, total=False):
|
||||
|
|
@ -713,6 +715,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
type="text",
|
||||
text="",
|
||||
)
|
||||
self._unsigned_thinking_block_index: int | None = None
|
||||
|
||||
@property
|
||||
def chunks(self) -> "list[ModelResponseStream] | None":
|
||||
|
|
@ -915,7 +918,36 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
except Exception: # noqa: BLE001 # prompt estimation must not interrupt the stream
|
||||
return 0
|
||||
|
||||
def __next__(self):
|
||||
def __next__(self) -> _AnthropicStreamEvent:
|
||||
return self._signed_thinking_event(self._next_event())
|
||||
|
||||
async def __anext__(self) -> _AnthropicStreamEvent:
|
||||
return self._signed_thinking_event(await self._anext_event())
|
||||
|
||||
def _signed_thinking_event(self, event: _AnthropicStreamEvent) -> _AnthropicStreamEvent:
|
||||
match event:
|
||||
case {"type": "content_block_start", "index": int() as index, "content_block": {"type": "thinking"}}:
|
||||
self._unsigned_thinking_block_index = index
|
||||
return event
|
||||
case {"type": "content_block_delta", "index": int() as index, "delta": {"type": "signature_delta"}} if (
|
||||
index == self._unsigned_thinking_block_index
|
||||
):
|
||||
self._unsigned_thinking_block_index = None
|
||||
return event
|
||||
case {"type": "content_block_stop", "index": int() as index} if (
|
||||
index == self._unsigned_thinking_block_index
|
||||
):
|
||||
self._unsigned_thinking_block_index = None
|
||||
self.chunk_queue.appendleft(event)
|
||||
return ContentBlockDelta(
|
||||
type="content_block_delta",
|
||||
index=index,
|
||||
delta=ContentThinkingSignatureBlockDelta(type="signature_delta", signature=""),
|
||||
)
|
||||
case _:
|
||||
return event
|
||||
|
||||
def _next_event(self) -> _AnthropicStreamEvent:
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
try:
|
||||
|
|
@ -1150,7 +1182,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
verbose_logger.error("Anthropic Adapter - %s\n%s", e, traceback.format_exc())
|
||||
raise StopIteration
|
||||
|
||||
async def __anext__(self):
|
||||
async def _anext_event(self) -> _AnthropicStreamEvent:
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
try:
|
||||
|
|
@ -1388,13 +1420,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
This wrapper ensures dict chunks are SSE formatted with both event and data lines.
|
||||
"""
|
||||
for chunk in self:
|
||||
if isinstance(chunk, dict):
|
||||
event_type: str = str(chunk.get("type", "message"))
|
||||
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
|
||||
yield payload.encode()
|
||||
else:
|
||||
# For non-dict chunks, forward the original value unchanged
|
||||
yield chunk
|
||||
yield anthropic_sse_frame(chunk)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._combined_chunk_splitter.aclose()
|
||||
|
|
@ -1409,9 +1435,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
)
|
||||
try:
|
||||
async for chunk in self:
|
||||
if not isinstance(chunk, dict):
|
||||
yield chunk
|
||||
elif safeguard_results is None:
|
||||
if safeguard_results is None:
|
||||
yield anthropic_sse_frame(chunk)
|
||||
else:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(_SSE_EVENT.validate_python(chunk)))
|
||||
|
|
|
|||
|
|
@ -1539,7 +1539,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
AnthropicResponseContentBlockThinking(
|
||||
type="thinking",
|
||||
thinking=(str(thinking_value) if thinking_value is not None else ""),
|
||||
signature=(str(signature_value) if signature_value is not None else None),
|
||||
signature=(str(signature_value) if signature_value is not None else ""),
|
||||
).model_dump()
|
||||
)
|
||||
elif thinking_block.get("type") == "redacted_thinking":
|
||||
|
|
@ -1556,7 +1556,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
AnthropicResponseContentBlockThinking(
|
||||
type="thinking",
|
||||
thinking=str(choice.message.reasoning_content),
|
||||
signature=None,
|
||||
signature="",
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,14 +103,12 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
"delta": {"type": "thinking_delta", "thinking": thinking_text},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
|
||||
signature: Final = block_dict.get("signature", "")
|
||||
if signature:
|
||||
signature_delta: Final = {
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "signature_delta", "signature": signature},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
|
||||
signature_delta: Final = {
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "signature_delta", "signature": block_dict.get("signature") or ""},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
|
||||
|
||||
elif block_type in ("tool_use", "server_tool_use"):
|
||||
empty_input: Final[Mapping[str, object]] = {}
|
||||
|
|
|
|||
|
|
@ -294,16 +294,11 @@ def _anthropic_content_block_start_and_deltas(
|
|||
)
|
||||
case "thinking":
|
||||
signature: Final = block.get("signature")
|
||||
signature_deltas: Final = (
|
||||
({"signature": signature, "type": "signature_delta"},)
|
||||
if isinstance(signature, str) and signature
|
||||
else ()
|
||||
)
|
||||
return (
|
||||
{"thinking": "", "signature": "", "type": "thinking"},
|
||||
(
|
||||
{"thinking": block.get("thinking") or "", "type": "thinking_delta"},
|
||||
*signature_deltas,
|
||||
{"signature": signature if isinstance(signature, str) else "", "type": "signature_delta"},
|
||||
),
|
||||
)
|
||||
case "redacted_thinking":
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from ...common_utils import (
|
|||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
strip_unsignable_thinking_blocks_for_claude,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
from .mid_conversation_system import (
|
||||
|
|
@ -779,12 +780,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts
|
||||
)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
|
||||
base_model: Final = getattr(litellm_params, "base_model", None)
|
||||
anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
|
||||
messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
||||
drop_non_string_tool_result_texts(
|
||||
_MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages,
|
||||
getattr(litellm_params, "drop_params", None),
|
||||
)
|
||||
messages=strip_unsignable_thinking_blocks_for_claude(
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
||||
drop_non_string_tool_result_texts(
|
||||
_MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages,
|
||||
getattr(litellm_params, "drop_params", None),
|
||||
)
|
||||
),
|
||||
model=model,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
),
|
||||
max_tokens=max_tokens,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
response.output_text.delta -> content_block_delta (text_delta)
|
||||
response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator)
|
||||
response.reasoning_summary_text.delta -> content_block_delta (thinking_delta)
|
||||
response.reasoning_text.delta -> content_block_delta (thinking_delta)
|
||||
response.function_call_arguments.delta -> content_block_delta (input_json_delta)
|
||||
response.output_item.done -> content_block_delta (signature_delta) + content_block_stop
|
||||
response.completed -> message_delta + message_stop
|
||||
|
|
@ -203,32 +204,52 @@ class AnthropicResponsesStreamWrapper:
|
|||
def _field(source: object, name: str) -> object:
|
||||
return source.get(name) if isinstance(source, dict) else getattr(source, name, None)
|
||||
|
||||
def _replay_whole_reasoning_item(self, item: object, item_id: str | None) -> None:
|
||||
block: Final = LiteLLMAnthropicToResponsesAPIAdapter.thinking_block_from_reasoning_item(
|
||||
iter_items(self._field(item, "summary") or self._field(item, "content") or ()),
|
||||
self._field(item, "encrypted_content"),
|
||||
)
|
||||
if block is None:
|
||||
return
|
||||
if block.get("type") == "redacted_thinking":
|
||||
redacted_idx: Final = self._open_block(item_id, block)
|
||||
self._chunk_queue.append({"type": "content_block_stop", "index": redacted_idx})
|
||||
return
|
||||
block_idx: Final = self._open_block(item_id, {"type": "thinking", "thinking": "", "signature": ""})
|
||||
self._chunk_queue.extend(
|
||||
(
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "thinking_delta", "thinking": block.get("thinking")},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "signature_delta", "signature": block.get("signature")},
|
||||
},
|
||||
{"type": "content_block_stop", "index": block_idx},
|
||||
)
|
||||
)
|
||||
|
||||
def _close_reasoning_item(self, item: object, item_id: str | None) -> None:
|
||||
block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
|
||||
if block_idx < 0:
|
||||
self._replay_whole_reasoning_item(item, item_id)
|
||||
return
|
||||
encrypted_content: Final = self._field(item, "encrypted_content")
|
||||
signature: Final = (
|
||||
encrypted_reasoning_signature(encrypted_content)
|
||||
if isinstance(encrypted_content, str) and encrypted_content
|
||||
else None
|
||||
)
|
||||
if block_idx < 0 and signature is None:
|
||||
return
|
||||
if block_idx < 0:
|
||||
redacted_idx: Final = self._open_block(
|
||||
item_id,
|
||||
{"type": "redacted_thinking", "data": signature},
|
||||
)
|
||||
stop: Final[dict[str, object]] = {"type": "content_block_stop", "index": redacted_idx}
|
||||
self._chunk_queue.append(stop)
|
||||
return
|
||||
if signature is not None:
|
||||
self._chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "signature_delta", "signature": signature},
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "signature_delta", "signature": signature or ""},
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "content_block_stop", "index": block_idx})
|
||||
|
||||
def _process_event(self, event: object) -> None:
|
||||
|
|
@ -332,8 +353,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
)
|
||||
return
|
||||
|
||||
# ---- reasoning summary text delta ----
|
||||
if event_type == "response.reasoning_summary_text.delta":
|
||||
if event_type in ("response.reasoning_summary_text.delta", "response.reasoning_text.delta"):
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
|
||||
block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
|
||||
|
|
|
|||
|
|
@ -8,10 +8,12 @@ path used for OpenAI and Azure models.
|
|||
import json
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from itertools import groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
|
|
@ -65,6 +67,12 @@ RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _output_item_mapping(item: object) -> Mapping[str, object]:
|
||||
if isinstance(item, BaseModel):
|
||||
return item.model_dump()
|
||||
return as_str_mapping(item) or MappingProxyType({})
|
||||
|
||||
|
||||
class LiteLLMAnthropicToResponsesAPIAdapter:
|
||||
"""
|
||||
Converts Anthropic /v1/messages requests to OpenAI Responses API format and
|
||||
|
|
@ -248,7 +256,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
return str(getattr(part, "text", None) or "")
|
||||
|
||||
@classmethod
|
||||
def _thinking_block_from_reasoning_item(
|
||||
def thinking_block_from_reasoning_item(
|
||||
cls,
|
||||
summary: Iterable[object],
|
||||
encrypted_content: object,
|
||||
|
|
@ -266,7 +274,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
if not isinstance(encrypted_content, str) or not encrypted_content:
|
||||
if not text:
|
||||
return None
|
||||
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump()
|
||||
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature="").model_dump()
|
||||
signature: Final = encrypted_reasoning_signature(encrypted_content)
|
||||
if not text:
|
||||
return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump()
|
||||
|
|
@ -744,94 +752,52 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
"""
|
||||
Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse.
|
||||
"""
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
|
||||
content: Final[list[dict[str, object]]] = []
|
||||
stop_reason: AnthropicFinishReason = "end_turn"
|
||||
refusal_text: Final = responses_output_refusal_text(
|
||||
cast(Iterable[object], response.output) # cast-ok: output items re-validated per item
|
||||
)
|
||||
|
||||
for item in response.output:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content)
|
||||
if reasoning_block is not None:
|
||||
content.append(reasoning_block)
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for part in item.content:
|
||||
part_type = getattr(part, "type", None)
|
||||
for item in map(_output_item_mapping, response.output):
|
||||
item_type = item.get("type")
|
||||
if item_type == "message":
|
||||
for part in filter(None, map(as_str_mapping, iter_items(item.get("content") or ()))):
|
||||
part_type = part.get("type")
|
||||
if part_type == "output_text":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump()
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=str(part.get("text") or "")
|
||||
).model_dump()
|
||||
)
|
||||
elif part_type == "refusal":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=getattr(part, "refusal", "") or ""
|
||||
type="text", text=str(part.get("refusal") or "")
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
elif item_type == "reasoning":
|
||||
reasoning_block = self.thinking_block_from_reasoning_item(
|
||||
iter_items(item.get("summary") or item.get("content") or ()),
|
||||
item.get("encrypted_content"),
|
||||
)
|
||||
if reasoning_block is not None:
|
||||
content.append(reasoning_block)
|
||||
elif item_type == "function_call":
|
||||
try:
|
||||
input_data = json.loads(item.arguments) if item.arguments else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
input_data = _DECODED_JSON.validate_python(json.loads(str(item.get("arguments") or "{}")))
|
||||
except json.JSONDecodeError:
|
||||
input_data = {}
|
||||
content.append(
|
||||
AnthropicResponseContentBlockToolUse(
|
||||
type="tool_use",
|
||||
id=item.call_id or item.id or "",
|
||||
name=item.name,
|
||||
input=input_data,
|
||||
AnthropicResponseContentBlockToolUse.model_validate(
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": item.get("call_id") or item.get("id") or "",
|
||||
"name": item.get("name") or "",
|
||||
"input": input_data,
|
||||
}
|
||||
).model_dump(exclude_none=True)
|
||||
)
|
||||
stop_reason = "tool_use"
|
||||
|
||||
elif isinstance(item, dict):
|
||||
item_type = item.get("type")
|
||||
if item_type == "message":
|
||||
for part in item.get("content", ()):
|
||||
if isinstance(part, dict):
|
||||
part_type = part.get("type")
|
||||
if part_type == "output_text":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=part.get("text", "")
|
||||
).model_dump()
|
||||
)
|
||||
elif part_type == "refusal":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=part.get("refusal", "") or ""
|
||||
).model_dump()
|
||||
)
|
||||
elif item_type == "reasoning":
|
||||
reasoning_block = self._thinking_block_from_reasoning_item(
|
||||
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
|
||||
item.get("encrypted_content"),
|
||||
)
|
||||
if reasoning_block is not None:
|
||||
content.append(reasoning_block)
|
||||
elif item_type == "function_call":
|
||||
try:
|
||||
input_data = _DECODED_JSON.validate_python(json.loads(item.get("arguments", "{}")))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
input_data = {}
|
||||
content.append(
|
||||
AnthropicResponseContentBlockToolUse.model_validate(
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": item.get("call_id") or item.get("id", ""),
|
||||
"name": item.get("name", ""),
|
||||
"input": input_data,
|
||||
}
|
||||
).model_dump(exclude_none=True)
|
||||
)
|
||||
stop_reason = "tool_use"
|
||||
if response.status == "incomplete":
|
||||
stop_reason = "max_tokens"
|
||||
elif refusal_text is not None:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ use litellm_host::{
|
|||
protocol::UnobservedTail,
|
||||
};
|
||||
use litellm_http::Phases;
|
||||
use litellm_llms::anthropic::messages::transformation::is_claude_model;
|
||||
use litellm_llms::base_llm::{
|
||||
auth::{Authenticated, AzureAdTokenHosts, ValidatedEnvironment, resolve_auth},
|
||||
messages::{
|
||||
|
|
@ -324,6 +325,11 @@ async fn provider_call(
|
|||
let transform_context = MessagesTransformContext {
|
||||
drops_unsupported_params: shaping.settings.drops_unsupported_params,
|
||||
global_drop_params: shaping.settings.global_drop_params,
|
||||
claude_base_model: shaping
|
||||
.settings
|
||||
.base_model
|
||||
.as_deref()
|
||||
.is_some_and(is_claude_model),
|
||||
..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params)
|
||||
};
|
||||
let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params);
|
||||
|
|
|
|||
|
|
@ -105,6 +105,8 @@ pub struct MessagesSettings {
|
|||
#[serde(skip)]
|
||||
pub module_api_key: Option<SecretValue>,
|
||||
#[serde(default)]
|
||||
pub base_model: Option<String>,
|
||||
#[serde(default)]
|
||||
pub detached_drain_timeout: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub detached_drain_limit: Option<usize>,
|
||||
|
|
|
|||
|
|
@ -581,6 +581,50 @@ async fn additional_drop_params_remove_fields_before_sending(call: MessagesCall)
|
|||
assert_eq!(sent["top_k"], 3);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_base_model(Some("anthropic/claude-sonnet-4-5"), json!([]))]
|
||||
#[case::open_base_model(Some("deepseek/deepseek-v4-pro"), json!([{"type": "thinking", "thinking": "plan", "signature": ""}]))]
|
||||
#[case::no_base_model(None, json!([{"type": "thinking", "thinking": "plan", "signature": ""}]))]
|
||||
#[tokio::test]
|
||||
async fn unsigned_thinking_is_dropped_for_an_alias_whose_base_model_names_claude(
|
||||
call: MessagesCall,
|
||||
#[case] base_model: Option<&str>,
|
||||
#[case] sent_thinking: Value,
|
||||
) {
|
||||
let upstream = upstream([message_response()]).await;
|
||||
let history = json!([
|
||||
{"role": "user", "content": "plan it"},
|
||||
{"role": "assistant", "content": [{"type": "thinking", "thinking": "plan", "signature": ""}]},
|
||||
{"role": "user", "content": "go on"}
|
||||
]);
|
||||
|
||||
run_message(MessagesCall {
|
||||
api_key: Some("sk".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
base_model: base_model.map(str::to_owned),
|
||||
..MessagesSettings::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..with_fields(call, json!({"model": "prod-sonnet", "messages": history}))
|
||||
})
|
||||
.await;
|
||||
|
||||
let sent = only_request(&upstream).await.json();
|
||||
let thinking: Vec<Value> = sent["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|message| message["content"].as_array())
|
||||
.flatten()
|
||||
.filter(|block| block["type"] == "thinking")
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(Value::from(thinking), sent_thinking);
|
||||
}
|
||||
|
||||
fn sent_betas(request: &wiremock::Request) -> BetaSet {
|
||||
let [header] = <[&str; 1]>::try_from(request.header_values("anthropic-beta"))
|
||||
.unwrap_or_else(|values| panic!("expected one anthropic-beta header, got {values:?}"));
|
||||
|
|
|
|||
|
|
@ -554,6 +554,17 @@ pub fn strip_encrypted_reasoning_blocks(messages: Vec<Message>) -> Vec<Message>
|
|||
retain_blocks(messages, |block| !is_encrypted_reasoning_block(block))
|
||||
}
|
||||
|
||||
fn is_unsigned_thinking_block(block: &ContentBlock) -> bool {
|
||||
block.is_type(ContentBlockType::Thinking)
|
||||
&& block.signature.as_deref().is_none_or(str::is_empty)
|
||||
}
|
||||
|
||||
/// Python's `strip_unsignable_thinking_blocks_for_claude` once the encrypted strip has run: Claude
|
||||
/// answers 400 to a thinking block with no signature, so an open model's reasoning goes.
|
||||
pub fn strip_unsigned_thinking_blocks(messages: Vec<Message>) -> Vec<Message> {
|
||||
retain_blocks(messages, |block| !is_unsigned_thinking_block(block))
|
||||
}
|
||||
|
||||
/// Python's `strip_encrypted_reasoning_blocks_from_anthropic_messages`: the same strip, and a
|
||||
/// kept `redacted_thinking` block gets Claude's own `data` back from a Bedrock Converse wrapper.
|
||||
pub fn strip_encrypted_reasoning_blocks_restoring_redacted_data(
|
||||
|
|
@ -1107,6 +1118,25 @@ pub fn strip_raw_encrypted_reasoning_blocks(message: Value) -> Option<Value> {
|
|||
})
|
||||
}
|
||||
|
||||
fn is_raw_unsigned_thinking_block(block: &Value) -> bool {
|
||||
raw_block_type(block) == Some("thinking")
|
||||
&& block
|
||||
.get("signature")
|
||||
.and_then(Value::as_str)
|
||||
.is_none_or(str::is_empty)
|
||||
}
|
||||
|
||||
pub fn strip_raw_unsigned_thinking_blocks(message: Value) -> Option<Value> {
|
||||
with_raw_blocks(message, |blocks| {
|
||||
let kept: Vec<Value> = blocks
|
||||
.iter()
|
||||
.filter(|block| !is_raw_unsigned_thinking_block(block))
|
||||
.cloned()
|
||||
.collect();
|
||||
(kept == blocks || !kept.is_empty()).then_some(kept)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_anthropic_invalid_thinking_block_error(error_text: &str) -> bool {
|
||||
let lower = error_text.to_lowercase();
|
||||
if !lower.contains("thinking") {
|
||||
|
|
@ -1741,6 +1771,34 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty_signature(
|
||||
json!([{"type": "thinking", "thinking": "plan", "signature": ""}, unparsed_block()]),
|
||||
Some(json!([unparsed_block()]))
|
||||
)]
|
||||
#[case::non_string_signature(
|
||||
json!([{"type": "thinking", "thinking": "plan", "signature": 5}, unparsed_block()]),
|
||||
Some(json!([unparsed_block()]))
|
||||
)]
|
||||
#[case::signed(
|
||||
json!([{"type": "thinking", "thinking": "plan", "signature": "sig"}, unparsed_block()]),
|
||||
Some(json!([{"type": "thinking", "thinking": "plan", "signature": "sig"}, unparsed_block()]))
|
||||
)]
|
||||
#[case::redacted_kept(
|
||||
json!([{"type": "redacted_thinking", "data": "EmwK"}, unparsed_block()]),
|
||||
Some(json!([{"type": "redacted_thinking", "data": "EmwK"}, unparsed_block()]))
|
||||
)]
|
||||
#[case::only_unsigned(json!([{"type": "thinking", "thinking": "plan"}]), None)]
|
||||
fn strip_raw_unsigned_thinking_blocks_drops_what_claude_rejects(
|
||||
#[case] blocks: Value,
|
||||
#[case] expected: Option<Value>,
|
||||
) {
|
||||
assert_eq!(
|
||||
strip_raw_unsigned_thinking_blocks(turn(blocks)),
|
||||
expected.map(turn)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::string_content(json!({"role": "user", "content": 5}))]
|
||||
#[case::not_an_object(json!("hi"))]
|
||||
|
|
|
|||
|
|
@ -37,8 +37,9 @@ use crate::{
|
|||
minted_oauth_environment, optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta, strip_advisor_blocks,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data, strip_raw_advisor_blocks,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict,
|
||||
strip_raw_encrypted_reasoning_blocks, strip_raw_unsigned_thinking_blocks,
|
||||
strip_thinking_blocks_from_anthropic_messages_request, strip_unsigned_thinking_blocks,
|
||||
tools_without_strict,
|
||||
},
|
||||
base_llm::{
|
||||
auth::AuthScheme,
|
||||
|
|
@ -202,7 +203,7 @@ fn is_foundry_url(url: &str) -> bool {
|
|||
})
|
||||
}
|
||||
|
||||
pub(crate) fn is_claude_model(model: &str) -> bool {
|
||||
pub fn is_claude_model(model: &str) -> bool {
|
||||
model.to_lowercase().contains("claude")
|
||||
}
|
||||
|
||||
|
|
@ -345,12 +346,22 @@ pub(crate) fn transform_messages_request(
|
|||
} else {
|
||||
messages
|
||||
};
|
||||
Ok(MessagesRequest {
|
||||
messages: map_messages(
|
||||
let messages = map_messages(
|
||||
messages,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
);
|
||||
let messages = if context.claude_base_model || is_claude_model(&request.model) {
|
||||
map_messages(
|
||||
messages,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
),
|
||||
strip_unsigned_thinking_blocks,
|
||||
strip_raw_unsigned_thinking_blocks,
|
||||
)
|
||||
} else {
|
||||
messages
|
||||
};
|
||||
Ok(MessagesRequest {
|
||||
messages,
|
||||
params: MessagesOptionalParams {
|
||||
context_management,
|
||||
extra: without_null_safeguards(request.params.extra),
|
||||
|
|
@ -1317,6 +1328,58 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn tool_turn(opening: Option<Value>) -> Value {
|
||||
let pending: Vec<Value> = opening
|
||||
.into_iter()
|
||||
.chain([json!({"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}})])
|
||||
.collect();
|
||||
json!([
|
||||
{"role": "user", "content": "List the files."},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "thinking", "thinking": "claude plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
|
||||
{"type": "text", "text": "Draft."}
|
||||
]},
|
||||
{"role": "user", "content": "Now list them."},
|
||||
{"role": "assistant", "content": pending},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]}
|
||||
])
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty(json!({"type": "thinking", "thinking": "open model plan", "signature": ""}))]
|
||||
#[case::null(json!({"type": "thinking", "thinking": "open model plan", "signature": null}))]
|
||||
#[case::missing(json!({"type": "thinking", "thinking": "open model plan"}))]
|
||||
fn unsigned_thinking_never_reaches_claude_even_on_the_pending_tool_turn(
|
||||
#[case] unsigned: Value,
|
||||
unmapped: MessagesModelCapabilities,
|
||||
) {
|
||||
assert_eq!(
|
||||
transform(
|
||||
json!({"messages": tool_turn(Some(unsigned))}),
|
||||
unmapped,
|
||||
false
|
||||
),
|
||||
Ok(body(json!({"messages": tool_turn(None)})))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn unsigned_thinking_is_still_replayed_to_an_open_model(unmapped: MessagesModelCapabilities) {
|
||||
let messages = tool_turn(Some(
|
||||
json!({"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
));
|
||||
assert_eq!(
|
||||
transform(
|
||||
json!({"model": "deepseek-v4-pro", "messages": messages.clone()}),
|
||||
unmapped,
|
||||
false
|
||||
),
|
||||
Ok(body(
|
||||
json!({"model": "deepseek-v4-pro", "messages": messages})
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thinking_is_translated_with_the_context_budgets() {
|
||||
let context = MessagesTransformContext::with_lookup(
|
||||
|
|
@ -1360,6 +1423,7 @@ mod tests {
|
|||
drop_params: true,
|
||||
drops_unsupported_params: false,
|
||||
global_drop_params: false,
|
||||
claude_base_model: false,
|
||||
}
|
||||
);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -178,6 +178,7 @@ pub struct MessagesTransformContext {
|
|||
pub drop_params: bool,
|
||||
pub drops_unsupported_params: bool,
|
||||
pub global_drop_params: bool,
|
||||
pub claude_base_model: bool,
|
||||
}
|
||||
impl MessagesTransformContext {
|
||||
pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self {
|
||||
|
|
@ -199,6 +200,7 @@ impl MessagesTransformContext {
|
|||
drop_params,
|
||||
drops_unsupported_params: false,
|
||||
global_drop_params: false,
|
||||
claude_base_model: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -696,6 +696,7 @@ impl MessagesPythonHost {
|
|||
unencodable_request_header,
|
||||
module_api_key: azure.as_ref().and_then(|azure| azure.api_key.clone()),
|
||||
global_drop_params: global_drop_params(py)?,
|
||||
base_model: argument("base_model")?.and_then(|value| value.extract().ok()),
|
||||
..settings.settings
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -65,6 +65,11 @@ impl Router {
|
|||
additional_drop_params: additional_drop_params(
|
||||
&litellm_params.extra,
|
||||
),
|
||||
base_model: litellm_params
|
||||
.extra
|
||||
.get("base_model")
|
||||
.and_then(|value| value.as_str())
|
||||
.map(str::to_owned),
|
||||
..MessagesSettings::default()
|
||||
},
|
||||
capabilities: anthropic_model_capabilities(
|
||||
|
|
|
|||
|
|
@ -747,7 +747,7 @@ class AnthropicResponseContentBlockToolUse(LiteLLMBaseModel):
|
|||
class AnthropicResponseContentBlockThinking(LiteLLMBaseModel):
|
||||
type: Literal["thinking"]
|
||||
thinking: str
|
||||
signature: str | None
|
||||
signature: str
|
||||
|
||||
|
||||
class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -447,6 +447,7 @@ def test_messages_stream_opens_thinking_block_at_index_zero_for_reasoning_conten
|
|||
(0, {"type": "thinking", "thinking": "", "signature": ""}),
|
||||
(0, {"type": "thinking_delta", "thinking": "Let me "}),
|
||||
(0, {"type": "thinking_delta", "thinking": "think."}),
|
||||
(0, {"type": "signature_delta", "signature": ""}),
|
||||
(1, {"type": "text", "text": ""}),
|
||||
(1, {"type": "text_delta", "text": "Hello "}),
|
||||
(1, {"type": "text_delta", "text": "there"}),
|
||||
|
|
@ -456,6 +457,7 @@ def test_messages_stream_opens_thinking_block_at_index_zero_for_reasoning_conten
|
|||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
AnthropicAdapter,
|
||||
|
|
@ -3233,7 +3236,7 @@ def test_translate_openai_content_to_anthropic_reasoning_content_without_thinkin
|
|||
assert result[0]["type"] == "thinking"
|
||||
assert "Considering Letter Frequency" in result[0]["thinking"]
|
||||
assert "Calculating the Count" in result[0]["thinking"]
|
||||
assert result[0]["signature"] is None
|
||||
assert result[0]["signature"] == ""
|
||||
# Second block should be text block with content
|
||||
assert result[1]["type"] == "text"
|
||||
assert result[1]["text"] == 'There are **3** "r"s in the word strawberry.'
|
||||
|
|
@ -3311,7 +3314,7 @@ def test_translate_openai_response_to_anthropic_with_reasoning_content_only():
|
|||
# First block should be thinking
|
||||
assert anthropic_content[0]["type"] == "thinking"
|
||||
assert "Considering Letter Frequency" in anthropic_content[0]["thinking"]
|
||||
assert anthropic_content[0].get("signature") is None
|
||||
assert anthropic_content[0].get("signature") == ""
|
||||
|
||||
# Second block should be text
|
||||
assert anthropic_content[1]["type"] == "text"
|
||||
|
|
@ -6293,3 +6296,71 @@ async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item(
|
|||
("thinking", "Check the forecast.", encrypted_reasoning_signature("gAAA_second")),
|
||||
("text", "", None),
|
||||
]
|
||||
|
||||
|
||||
_TOOL_LOOP_FROM_AN_OPEN_MODEL: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "open model plan", "signature": ""},
|
||||
{"type": "text", "text": "Draft."},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "List the files."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "call ls", "signature": ""},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]},
|
||||
]
|
||||
|
||||
|
||||
def _sent_assistant_turns(model: str, deployment: Mapping[str, object]) -> list[dict[str, object]]:
|
||||
completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=_TOOL_LOOP_FROM_AN_OPEN_MODEL,
|
||||
model=model,
|
||||
thinking={"type": "enabled", "budget_tokens": 1024},
|
||||
extra_kwargs=deployment,
|
||||
)
|
||||
return [m for m in completion_kwargs["messages"] if m["role"] == "assistant"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, deployment",
|
||||
[
|
||||
pytest.param(
|
||||
"bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
{"custom_llm_provider": "bedrock"},
|
||||
id="bedrock-converse",
|
||||
),
|
||||
pytest.param(
|
||||
"github_copilot/claude-sonnet-4.5",
|
||||
{"custom_llm_provider": "github_copilot"},
|
||||
id="github-copilot",
|
||||
),
|
||||
pytest.param(
|
||||
"bedrock/converse/arn:aws:bedrock:us-west-2:123456789012:application-inference-profile/abc123",
|
||||
{"custom_llm_provider": "bedrock", "base_model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"},
|
||||
id="application-inference-profile-with-base-model",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_unsigned_thinking_never_reaches_claude_through_the_chat_adapter(
|
||||
model: str, deployment: Mapping[str, object]
|
||||
) -> None:
|
||||
turns: Final = _sent_assistant_turns(model, deployment)
|
||||
|
||||
assert [turn.get("thinking_blocks") for turn in turns] == [None, None]
|
||||
assert [turn.get("reasoning_content") for turn in turns] == [None, None]
|
||||
assert turns[1]["tool_calls"][0]["id"] == "toolu_1"
|
||||
|
||||
|
||||
def test_unsigned_thinking_is_still_replayed_to_an_open_model_through_the_chat_adapter() -> None:
|
||||
turns: Final = _sent_assistant_turns("deepseek/deepseek-v4-pro", {"custom_llm_provider": "deepseek"})
|
||||
|
||||
assert [turn.get("reasoning_content") for turn in turns] == ["open model plan", "call ls"]
|
||||
|
|
|
|||
|
|
@ -1534,3 +1534,56 @@ async def test_signed_thinking_blocks_keep_their_own_text_and_signature(is_async
|
|||
assert _last_delta_types_per_thinking_block(events) == ("signature_delta", "signature_delta")
|
||||
assert _text_deltas(events) == ["Done"]
|
||||
_assert_deltas_match_their_block_type(events)
|
||||
|
||||
|
||||
_UNSIGNED_REASONING_STREAMS: Final[Mapping[str, Tuple[MagicMock, ...]]] = MappingProxyType(
|
||||
{
|
||||
"then-text": (
|
||||
_reasoning_content_chunk("The user "),
|
||||
_reasoning_content_chunk("wants 2+2."),
|
||||
_make_chunk(Delta(content="4")),
|
||||
_make_chunk(Delta(content=None), finish_reason="stop"),
|
||||
),
|
||||
"then-tool-call": (
|
||||
_reasoning_content_chunk("The user wants 2+2."),
|
||||
_tool_chunk("call_add", "add", '{"a": 2, "b": 2}'),
|
||||
_make_chunk(Delta(content=None), finish_reason="tool_calls"),
|
||||
),
|
||||
"thought-ends-with-finish": (
|
||||
_reasoning_content_chunk("The user "),
|
||||
_make_chunk(Delta(content=None, reasoning_content="wants 2+2."), finish_reason="stop"),
|
||||
),
|
||||
"unsigned-thinking-blocks": (
|
||||
_thinking_chunk("The user "),
|
||||
_thinking_chunk("wants 2+2."),
|
||||
_make_chunk(Delta(content="4")),
|
||||
_make_chunk(Delta(content=None), finish_reason="stop"),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _event_before_stop(events: Sequence[Mapping[str, object]], index: int) -> Mapping[str, object]:
|
||||
stop_position: Final = next(
|
||||
position
|
||||
for position, event in enumerate(events)
|
||||
if event.get("type") == "content_block_stop" and event["index"] == index
|
||||
)
|
||||
return events[stop_position - 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("stream", list(_UNSIGNED_REASONING_STREAMS), ids=list(_UNSIGNED_REASONING_STREAMS))
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsigned_thinking_block_closes_with_an_empty_signature_delta(is_async: bool, stream: str):
|
||||
events: Final = await _events_for(_UNSIGNED_REASONING_STREAMS[stream], is_async)
|
||||
|
||||
assert _thinking_blocks_as_accumulated(events) == (
|
||||
_AccumulatedThinking(thinking="The user wants 2+2.", signature=""),
|
||||
)
|
||||
assert [_event_before_stop(events, index) for index in _thinking_starts(events)] == [
|
||||
{"type": "content_block_delta", "index": index, "delta": {"type": "signature_delta", "signature": ""}}
|
||||
for index in _thinking_starts(events)
|
||||
]
|
||||
assert _signature_deltas(events) == [""]
|
||||
_assert_deltas_match_their_block_type(events)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
def _transform(messages):
|
||||
|
|
@ -48,3 +54,126 @@ def test_anthropic_signed_thinking_blocks_are_forwarded_untouched():
|
|||
},
|
||||
]
|
||||
assert _transform(messages)["messages"] == messages
|
||||
|
||||
|
||||
_CLAUDE_SIGNED_TURN: Final = {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "claude plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
|
||||
{"type": "text", "text": "Refined."},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _open_model_turn(thinking_block: dict[str, object]) -> dict[str, object]:
|
||||
return {"role": "assistant", "content": [thinking_block, {"type": "text", "text": "Draft."}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_block",
|
||||
[
|
||||
{"type": "thinking", "thinking": "open model plan", "signature": ""},
|
||||
{"type": "thinking", "thinking": "open model plan", "signature": None},
|
||||
{"type": "thinking", "thinking": "open model plan"},
|
||||
],
|
||||
ids=["empty", "null", "missing"],
|
||||
)
|
||||
def test_unsigned_thinking_from_an_open_model_never_reaches_claude(thinking_block: dict[str, object]) -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
_open_model_turn(thinking_block),
|
||||
{"role": "user", "content": "Refine it."},
|
||||
_CLAUDE_SIGNED_TURN,
|
||||
{"role": "user", "content": "Thanks."},
|
||||
]
|
||||
|
||||
sent: Final = _transform(messages)["messages"]
|
||||
|
||||
assert sent[1]["content"] == [{"type": "text", "text": "Draft."}]
|
||||
assert sent[3] == _CLAUDE_SIGNED_TURN
|
||||
assert messages[1]["content"][0] == thinking_block
|
||||
|
||||
|
||||
def test_unsigned_thinking_is_still_replayed_to_an_open_model() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
_open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
{"role": "user", "content": "Refine it."},
|
||||
]
|
||||
|
||||
sent: Final = DeepSeekAnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="deepseek-v4-pro",
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params={"max_tokens": 1024},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)["messages"]
|
||||
|
||||
assert sent[1]["content"][0]["thinking"] == "open model plan"
|
||||
|
||||
|
||||
def _pending_tool_turn(opening: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "user", "content": "List the files."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [*opening, {"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}}],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]},
|
||||
]
|
||||
|
||||
|
||||
def test_unsigned_thinking_that_opens_the_pending_tool_turn_is_dropped_too() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
_open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
*_pending_tool_turn([{"type": "thinking", "thinking": "call ls", "signature": ""}]),
|
||||
]
|
||||
|
||||
sent: Final = _transform(messages)["messages"]
|
||||
|
||||
assert sent[1]["content"] == [{"type": "text", "text": "Draft."}]
|
||||
assert sent[3]["content"] == [{"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}}]
|
||||
assert sent[4] == messages[4]
|
||||
|
||||
|
||||
def test_unsigned_thinking_is_dropped_when_the_pending_tool_turn_never_opened_with_thinking() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
_open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
*_pending_tool_turn([]),
|
||||
]
|
||||
|
||||
sent: Final = _transform(messages)["messages"]
|
||||
|
||||
assert sent[1]["content"] == [{"type": "text", "text": "Draft."}]
|
||||
assert sent[2:] == messages[2:]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_model", "sent_opening"),
|
||||
[
|
||||
("anthropic/claude-sonnet-4-5", {"type": "text", "text": "Draft."}),
|
||||
("deepseek/deepseek-v4-pro", {"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
(None, {"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
],
|
||||
ids=["claude-base-model", "open-base-model", "no-base-model"],
|
||||
)
|
||||
def test_an_alias_strips_unsigned_thinking_only_when_its_base_model_names_claude(
|
||||
base_model: str | None, sent_opening: dict[str, object]
|
||||
) -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Solve it."},
|
||||
_open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}),
|
||||
{"role": "user", "content": "Refine it."},
|
||||
]
|
||||
|
||||
sent: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="prod-sonnet",
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params={"max_tokens": 1024},
|
||||
litellm_params=GenericLiteLLMParams(base_model=base_model),
|
||||
headers={},
|
||||
)["messages"]
|
||||
|
||||
assert sent[1]["content"][0] == sent_opening
|
||||
|
|
|
|||
|
|
@ -232,3 +232,32 @@ def _sdk_rebuilt_content(block: Mapping[str, object]) -> list[dict[str, object]]
|
|||
@pytest.mark.parametrize("block", _REPLAYED_BLOCKS, ids=lambda block: str(block["type"]))
|
||||
def test_anthropic_sdk_rebuilds_every_block_field_from_the_stream(block: Mapping[str, object]):
|
||||
assert _sdk_rebuilt_content(block) == [block]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsigned_block",
|
||||
[
|
||||
{"type": "thinking", "thinking": "Let me think.", "signature": ""},
|
||||
{"type": "thinking", "thinking": "Let me think.", "signature": None},
|
||||
{"type": "thinking", "thinking": "Let me think."},
|
||||
],
|
||||
ids=["empty", "null", "missing"],
|
||||
)
|
||||
def test_unsigned_thinking_block_closes_with_an_empty_signature_delta(unsigned_block: Mapping[str, object]):
|
||||
response: Final = {
|
||||
"id": "msg_unsigned",
|
||||
"content": [unsigned_block, {"type": "text", "text": "Hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
thinking_events: Final = tuple(
|
||||
(event_type, payload.get("delta", payload.get("content_block")))
|
||||
for event_type, payload in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response))
|
||||
if payload.get("index") == 0
|
||||
)
|
||||
assert thinking_events == (
|
||||
("content_block_start", {"type": "thinking", "thinking": "", "signature": ""}),
|
||||
("content_block_delta", {"type": "thinking_delta", "thinking": "Let me think."}),
|
||||
("content_block_delta", {"type": "signature_delta", "signature": ""}),
|
||||
("content_block_stop", None),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1372,15 +1372,25 @@ def test_anthropic_messages_response_as_sse_events_thinking_block_emits_signatur
|
|||
]
|
||||
|
||||
|
||||
def test_anthropic_messages_response_as_sse_events_thinking_block_without_signature_omits_delta():
|
||||
response = {
|
||||
"id": "msg_6",
|
||||
"content": [{"type": "thinking", "thinking": "let me think", "signature": None}],
|
||||
"stop_reason": "end_turn",
|
||||
}
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_block",
|
||||
[
|
||||
{"type": "thinking", "thinking": "let me think", "signature": None},
|
||||
{"type": "thinking", "thinking": "let me think", "signature": ""},
|
||||
{"type": "thinking", "thinking": "let me think"},
|
||||
],
|
||||
ids=["null", "empty", "missing"],
|
||||
)
|
||||
def test_anthropic_messages_response_as_sse_events_unsigned_thinking_block_closes_with_an_empty_signature(
|
||||
thinking_block,
|
||||
):
|
||||
response = {"id": "msg_6", "content": [thinking_block], "stop_reason": "end_turn"}
|
||||
decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
|
||||
deltas = [payload["delta"] for event_type, payload in decoded if event_type == "content_block_delta"]
|
||||
assert deltas == [{"type": "thinking_delta", "thinking": "let me think"}]
|
||||
assert deltas == [
|
||||
{"type": "thinking_delta", "thinking": "let me think"},
|
||||
{"type": "signature_delta", "signature": ""},
|
||||
]
|
||||
|
||||
|
||||
def test_anthropic_messages_response_as_sse_events_multiple_blocks_are_indexed():
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ class TestReasoningItemWithoutSummaryText:
|
|||
("content_block_start", 0),
|
||||
("content_block_delta", 0),
|
||||
("content_block_delta", 0),
|
||||
("content_block_delta", 0),
|
||||
("content_block_stop", 0),
|
||||
("content_block_start", 1),
|
||||
("content_block_delta", 1),
|
||||
|
|
@ -190,10 +191,73 @@ class TestReasoningItemWithoutSummaryText:
|
|||
assert "".join(c["delta"]["thinking"] for c in chunks[2:4]) == "Weighing options"
|
||||
|
||||
def test_the_reasoning_item_id_is_never_streamed_as_a_signature(self):
|
||||
"""A stand-in signature would be replayed as a real one, so none is ever sent."""
|
||||
"""A stand-in signature would be replayed as a real one, so the block closes with an empty one."""
|
||||
chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["Weighing options"]))
|
||||
|
||||
assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"]
|
||||
signature_deltas: Final = [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"]
|
||||
assert [(c["index"], c["delta"]["signature"]) for c in signature_deltas] == [(0, "")]
|
||||
assert chunks[chunks.index(signature_deltas[0]) + 1] == {"type": "content_block_stop", "index": 0}
|
||||
|
||||
|
||||
class TestRawReasoningTextIsStreamedAsThinking:
|
||||
"""Servers that return a reasoning item's raw text (vLLM and OpenRouter for gpt-oss)
|
||||
stream it as ``response.reasoning_text.delta`` instead of a summary."""
|
||||
|
||||
def test_reasoning_text_deltas_become_one_closed_thinking_block(self):
|
||||
chunks: Final = _drain_async(
|
||||
[
|
||||
{"type": "response.created"},
|
||||
{"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
{"type": "response.reasoning_text.delta", "item_id": "rs_1", "content_index": 0, "delta": "17*23"},
|
||||
{"type": "response.reasoning_text.delta", "item_id": "rs_1", "content_index": 0, "delta": " is 391."},
|
||||
{"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}},
|
||||
{"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}},
|
||||
{"type": "response.output_text.delta", "item_id": "msg_1", "delta": "391"},
|
||||
{"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}},
|
||||
{"type": "response.completed"},
|
||||
]
|
||||
)
|
||||
|
||||
assert [(c["type"], c.get("index"), c.get("content_block") or c.get("delta")) for c in chunks[1:-2]] == [
|
||||
("content_block_start", 0, {"type": "thinking", "thinking": "", "signature": ""}),
|
||||
("content_block_delta", 0, {"type": "thinking_delta", "thinking": "17*23"}),
|
||||
("content_block_delta", 0, {"type": "thinking_delta", "thinking": " is 391."}),
|
||||
("content_block_delta", 0, {"type": "signature_delta", "signature": ""}),
|
||||
("content_block_stop", 0, None),
|
||||
("content_block_start", 1, {"type": "text", "text": ""}),
|
||||
("content_block_delta", 1, {"type": "text_delta", "text": "391"}),
|
||||
("content_block_stop", 1, None),
|
||||
]
|
||||
|
||||
def test_a_whole_reasoning_item_without_deltas_becomes_one_closed_thinking_block(self):
|
||||
reasoning_item: Final = {
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"summary": [],
|
||||
"content": [{"type": "reasoning_text", "text": "Just answer 391."}],
|
||||
"encrypted_content": None,
|
||||
}
|
||||
chunks: Final = _drain_async(
|
||||
[
|
||||
{"type": "response.created"},
|
||||
{"type": "response.output_item.added", "item": reasoning_item},
|
||||
{"type": "response.output_item.done", "item": reasoning_item},
|
||||
{"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}},
|
||||
{"type": "response.output_text.delta", "item_id": "msg_1", "delta": "391"},
|
||||
{"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}},
|
||||
{"type": "response.completed"},
|
||||
]
|
||||
)
|
||||
|
||||
assert [(c["type"], c.get("index"), c.get("content_block") or c.get("delta")) for c in chunks[1:-2]] == [
|
||||
("content_block_start", 0, {"type": "thinking", "thinking": "", "signature": ""}),
|
||||
("content_block_delta", 0, {"type": "thinking_delta", "thinking": "Just answer 391."}),
|
||||
("content_block_delta", 0, {"type": "signature_delta", "signature": ""}),
|
||||
("content_block_stop", 0, None),
|
||||
("content_block_start", 1, {"type": "text", "text": ""}),
|
||||
("content_block_delta", 1, {"type": "text_delta", "text": "391"}),
|
||||
("content_block_stop", 1, None),
|
||||
]
|
||||
|
||||
|
||||
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,14 @@ from typing import Any, Dict, Final, List, Literal
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseInputImageParam
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseInputImageParam,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
from openai.types.responses.response_reasoning_item import Summary
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.constants import (
|
||||
|
|
@ -33,11 +40,15 @@ from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
|||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponseAPIUsage
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
|
||||
def _make_request(**overrides) -> AnthropicMessagesRequest:
|
||||
|
|
@ -1410,20 +1421,14 @@ def _make_mock_response(
|
|||
return resp
|
||||
|
||||
|
||||
def _make_output_message(texts: List[str]) -> MagicMock:
|
||||
"""Build a mock ResponseOutputMessage with output_text parts."""
|
||||
from openai.types.responses import ResponseOutputMessage # type: ignore[import]
|
||||
|
||||
parts = []
|
||||
for t in texts:
|
||||
part = MagicMock()
|
||||
part.type = "output_text"
|
||||
part.text = t
|
||||
parts.append(part)
|
||||
|
||||
msg = MagicMock(spec=ResponseOutputMessage)
|
||||
msg.content = parts
|
||||
return msg
|
||||
def _make_output_message(texts: List[str]) -> ResponseOutputMessage:
|
||||
return ResponseOutputMessage(
|
||||
id="msg_test_1",
|
||||
content=[ResponseOutputText(type="output_text", text=text, annotations=[]) for text in texts],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
)
|
||||
|
||||
|
||||
def _make_refusal_message(refusal_text: str):
|
||||
|
|
@ -1438,35 +1443,19 @@ def _make_refusal_message(refusal_text: str):
|
|||
)
|
||||
|
||||
|
||||
def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock:
|
||||
"""Build a mock ResponseFunctionToolCall."""
|
||||
from openai.types.responses import ResponseFunctionToolCall # type: ignore[import]
|
||||
|
||||
item = MagicMock(spec=ResponseFunctionToolCall)
|
||||
item.call_id = call_id
|
||||
item.id = call_id
|
||||
item.name = name
|
||||
item.arguments = arguments
|
||||
return item
|
||||
def _make_function_call_item(call_id: str, name: str, arguments: str) -> ResponseFunctionToolCall:
|
||||
return ResponseFunctionToolCall(call_id=call_id, id=call_id, name=name, arguments=arguments, type="function_call")
|
||||
|
||||
|
||||
def _make_reasoning_item(
|
||||
summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None
|
||||
) -> MagicMock:
|
||||
"""Build a mock ResponseReasoningItem."""
|
||||
from openai.types.responses import ResponseReasoningItem # type: ignore[import]
|
||||
|
||||
summary_mocks = []
|
||||
for text in summaries:
|
||||
s = MagicMock()
|
||||
s.text = text
|
||||
summary_mocks.append(s)
|
||||
|
||||
item = MagicMock(spec=ResponseReasoningItem)
|
||||
item.id = item_id
|
||||
item.summary = summary_mocks
|
||||
item.encrypted_content = encrypted_content
|
||||
return item
|
||||
) -> ResponseReasoningItem:
|
||||
return ResponseReasoningItem(
|
||||
id=item_id,
|
||||
summary=[Summary(text=text, type="summary_text") for text in summaries],
|
||||
type="reasoning",
|
||||
encrypted_content=encrypted_content,
|
||||
)
|
||||
|
||||
|
||||
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
|
||||
|
|
@ -1596,7 +1585,7 @@ class TestTranslateResponse:
|
|||
reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123")
|
||||
response = _make_mock_response(output=[reasoning])
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert [block["signature"] for block in result["content"]] == [None]
|
||||
assert [block["signature"] for block in result["content"]] == [""]
|
||||
assert "rs_abc123" not in json.dumps(result["content"])
|
||||
|
||||
def test_summary_parts_join_into_one_thinking_block(self):
|
||||
|
|
@ -1684,7 +1673,7 @@ class TestTranslateResponse:
|
|||
]
|
||||
)
|
||||
result: Any = _ADAPTER.translate_response(response)
|
||||
assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}]
|
||||
assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": ""}]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("summaries", "encrypted_content"),
|
||||
|
|
@ -2530,3 +2519,51 @@ class TestPromptCacheBreakpointToResponses:
|
|||
extra_kwargs={"prompt_cache_options": {"mode": "explicit"}},
|
||||
)
|
||||
assert kwargs["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
class TestTranslateChatBridgedResponse:
|
||||
@staticmethod
|
||||
def _bridged(message: Message, finish_reason: str) -> ResponsesAPIResponse:
|
||||
return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
request_input="Say OK",
|
||||
responses_api_request={},
|
||||
chat_completion_response=ModelResponse(
|
||||
model="deepseek-v4-pro", choices=[Choices(index=0, finish_reason=finish_reason, message=message)]
|
||||
),
|
||||
)
|
||||
|
||||
def test_reasoning_and_text_are_kept(self):
|
||||
response: Final = self._bridged(
|
||||
Message(role="assistant", content="OK", reasoning_content="The user wants OK."), "stop"
|
||||
)
|
||||
|
||||
result: Final = _ADAPTER.translate_response(response)
|
||||
|
||||
assert result["content"] == [
|
||||
{"type": "thinking", "thinking": "The user wants OK.", "signature": ""},
|
||||
{"type": "text", "text": "OK"},
|
||||
]
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
def test_tool_call_is_kept(self):
|
||||
response: Final = self._bridged(
|
||||
Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Oslo"}'},
|
||||
}
|
||||
],
|
||||
),
|
||||
"tool_calls",
|
||||
)
|
||||
|
||||
result: Final = _ADAPTER.translate_response(response)
|
||||
|
||||
assert result["content"] == [
|
||||
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}
|
||||
]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
|
|
|||
|
|
@ -265,6 +265,29 @@ def test_request_transform_keeps_the_billing_header_block() -> None:
|
|||
assert request["system"] == [billing_block, {"type": "text", "text": "be brief"}]
|
||||
|
||||
|
||||
|
||||
def test_request_transform_drops_thinking_claude_cannot_verify() -> None:
|
||||
request: Final = DatabricksAnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model=CLAUDE_MODEL,
|
||||
messages=[
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "open model plan", "signature": ""},
|
||||
{"type": "text", "text": "reply"},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "second"},
|
||||
],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 64},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"][1]["content"] == [{"type": "text", "text": "reply"}]
|
||||
|
||||
|
||||
def _reply(request: httpx.Request, body: dict[str, object]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
|
|
|
|||
|
|
@ -360,3 +360,47 @@ class TestContentBlockState:
|
|||
]
|
||||
assert BLOCK_MESSAGE in raw
|
||||
assert '"index": 1' in raw
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_so_far, closing_deltas",
|
||||
[
|
||||
((), [{"type": "signature_delta", "signature": ""}]),
|
||||
(
|
||||
({"type": "signature_delta", "signature": "sig-abc"},),
|
||||
[],
|
||||
),
|
||||
],
|
||||
ids=["unsigned", "already-signed"],
|
||||
)
|
||||
def test_continuation_signs_an_open_thinking_block_once(self, thinking_so_far, closing_deltas):
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
|
||||
seen = [
|
||||
_sse_event(
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
|
||||
},
|
||||
),
|
||||
_sse_event(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "Hm"}},
|
||||
),
|
||||
*(
|
||||
_sse_event("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": delta})
|
||||
for delta in thinking_so_far
|
||||
),
|
||||
]
|
||||
exc = ModifyResponseException(
|
||||
message=BLOCK_MESSAGE, model="claude-3-5-sonnet", request_data={}, guardrail_name="g"
|
||||
)
|
||||
|
||||
raw = b"".join(
|
||||
self._handler().build_block_sse_chunks(exc, stream_started=True, responses_so_far=seen)
|
||||
).decode()
|
||||
payloads = _parse_sse_payloads(raw)
|
||||
stop_position = payloads.index({"type": "content_block_stop", "index": 0})
|
||||
|
||||
assert [payload["delta"] for payload in payloads[:stop_position]] == closing_deltas
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue