diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index ca9529ba246..4624d2d0524 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -574,6 +574,11 @@ def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_lin return None +def _mapping_type(value: object) -> object: + mapping: Final = as_str_mapping(value) + return mapping.get("type") if mapping is not None else None + + class AnthropicMessagesHandler(BaseTranslation): """Process Anthropic messages with guardrails. @@ -698,6 +703,17 @@ class AnthropicMessagesHandler(BaseTranslation): open_index, max_index = self._content_block_state(responses_so_far) new_index: Final = (max_index + 1) if max_index is not None else 0 chunks: list[bytes] = [] + if open_index is not None and self._is_unsigned_thinking_block(responses_so_far, open_index): + chunks.append( + _sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": open_index, + "delta": {"type": "signature_delta", "signature": ""}, + }, + ) + ) if open_index is not None: chunks.append(_sse("content_block_stop", {"type": "content_block_stop", "index": open_index})) chunks += [ @@ -730,6 +746,18 @@ class AnthropicMessagesHandler(BaseTranslation): ] return chunks + @staticmethod + def _is_unsigned_thinking_block(responses_so_far: Sequence[object], index: int) -> bool: + events: Final = tuple( + event + for event in chain.from_iterable(map(AnthropicMessagesHandler._iter_sse_events, responses_so_far)) + if event.get("index") == index + ) + return any( + event.get("type") == "content_block_start" and _mapping_type(event.get("content_block")) == "thinking" + for event in events + ) and not any(_mapping_type(event.get("delta")) == "signature_delta" for event in events) + @staticmethod def _content_block_state( responses_so_far: Sequence[object], diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index b753fe03b5a..b26aa91b573 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -28,7 +28,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, lax_bool from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( get_file_ids_from_messages, @@ -41,6 +41,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( is_encrypted_reasoning_block, + is_unsignable_thinking_block, native_redacted_thinking_data, ) from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error @@ -2187,6 +2188,36 @@ def strip_encrypted_reasoning_blocks_from_anthropic_messages( return [m for m in stripped if m is not None] +def _without_unsignable_thinking( + message: dict[str, object], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] +) -> dict[str, object] | None: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + fields: Final = as_str_mapping(message) + content: Final = as_object_list(fields.get("content")) if fields is not None else None + if content is None: + return message + kept: Final = [block for block in content if not is_unsignable_thinking_block(block)] + if len(kept) == len(content): + return message + return {**message, "content": kept} if kept else None + + +def strip_unsignable_thinking_blocks_for_claude( + messages: list[dict[str, object]], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + model: str, + base_model: str | None = None, +) -> list[dict[str, object]]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + """ + Drop thinking blocks Claude cannot verify (no signature, or another provider's + encrypted reasoning) before /v1/messages sends the request to a Claude model on + the native or chat adapter route, since Claude answers 400 to them, a pending + tool turn included. Other targets (DeepSeek, MiniMax, GLM) need their unsigned + reasoning replayed. + """ + if not any("claude" in name.lower() for name in (model, base_model or "")): + return messages + return [m for m in map(_without_unsignable_thinking, messages) if m is not None] + + def strip_thinking_blocks_from_anthropic_messages_request_dict( data: dict[str, object], ) -> None: diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 70b539a14d4..d79ffdcfb21 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -29,7 +29,10 @@ from litellm.litellm_core_utils.tool_fields import ( drops_mcp_servers, tools_without_mcp_servers, ) -from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages +from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + strip_unsignable_thinking_blocks_for_claude, +) from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, openai_thinking_routes_to_responses, @@ -744,6 +747,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) provider_kwargs: Final = extra_kwargs or {} + base_model: Final = provider_kwargs.get("base_model") custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider") target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None drop_params: Final = drops_unsupported_params(provider_kwargs.get("drop_params")) @@ -762,7 +766,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data: Final[dict[str, object]] = { "model": model, - "messages": messages, + "messages": strip_unsignable_thinking_blocks_for_claude( + messages, model, base_model if isinstance(base_model, str) else None + ), "max_tokens": max_tokens, } diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 343ce85dcd1..3b3e909b468 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -3,7 +3,6 @@ import copy import io import itertools -import json import traceback from collections import deque from collections.abc import AsyncIterator, Iterator, Mapping, Sequence @@ -15,6 +14,7 @@ from typing import ( Final, Literal, Protocol, + TypeAlias, cast, get_args, runtime_checkable, @@ -42,6 +42,7 @@ from litellm.types.llms.anthropic import ( AppliedEdit, CompactionBlock, ContentBlockDelta, + ContentThinkingSignatureBlockDelta, ContextManagementResponse, MessageBlockDelta, MessageDelta, @@ -68,6 +69,7 @@ _SSE_EVENT: Final = TypeAdapter(Mapping[str, object]) _THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock) _PROMPT_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) _PROMPT_TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam]) +_AnthropicStreamEvent: TypeAlias = dict[str, object] | MessageBlockDelta | ContentBlockDelta class _UsageDeltaWithIterations(UsageDelta, total=False): @@ -713,6 +715,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): type="text", text="", ) + self._unsigned_thinking_block_index: int | None = None @property def chunks(self) -> "list[ModelResponseStream] | None": @@ -915,7 +918,36 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): except Exception: # noqa: BLE001 # prompt estimation must not interrupt the stream return 0 - def __next__(self): + def __next__(self) -> _AnthropicStreamEvent: + return self._signed_thinking_event(self._next_event()) + + async def __anext__(self) -> _AnthropicStreamEvent: + return self._signed_thinking_event(await self._anext_event()) + + def _signed_thinking_event(self, event: _AnthropicStreamEvent) -> _AnthropicStreamEvent: + match event: + case {"type": "content_block_start", "index": int() as index, "content_block": {"type": "thinking"}}: + self._unsigned_thinking_block_index = index + return event + case {"type": "content_block_delta", "index": int() as index, "delta": {"type": "signature_delta"}} if ( + index == self._unsigned_thinking_block_index + ): + self._unsigned_thinking_block_index = None + return event + case {"type": "content_block_stop", "index": int() as index} if ( + index == self._unsigned_thinking_block_index + ): + self._unsigned_thinking_block_index = None + self.chunk_queue.appendleft(event) + return ContentBlockDelta( + type="content_block_delta", + index=index, + delta=ContentThinkingSignatureBlockDelta(type="signature_delta", signature=""), + ) + case _: + return event + + def _next_event(self) -> _AnthropicStreamEvent: from .transformation import LiteLLMAnthropicMessagesAdapter try: @@ -1150,7 +1182,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): verbose_logger.error("Anthropic Adapter - %s\n%s", e, traceback.format_exc()) raise StopIteration - async def __anext__(self): + async def _anext_event(self) -> _AnthropicStreamEvent: from .transformation import LiteLLMAnthropicMessagesAdapter try: @@ -1388,13 +1420,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): This wrapper ensures dict chunks are SSE formatted with both event and data lines. """ for chunk in self: - if isinstance(chunk, dict): - event_type: str = str(chunk.get("type", "message")) - payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" - yield payload.encode() - else: - # For non-dict chunks, forward the original value unchanged - yield chunk + yield anthropic_sse_frame(chunk) async def aclose(self) -> None: await self._combined_chunk_splitter.aclose() @@ -1409,9 +1435,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ) try: async for chunk in self: - if not isinstance(chunk, dict): - yield chunk - elif safeguard_results is None: + if safeguard_results is None: yield anthropic_sse_frame(chunk) else: yield anthropic_sse_frame(await safeguard_results.observe(_SSE_EVENT.validate_python(chunk))) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 660c7c1dbc9..38977280dd3 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -1539,7 +1539,7 @@ class LiteLLMAnthropicMessagesAdapter: AnthropicResponseContentBlockThinking( type="thinking", thinking=(str(thinking_value) if thinking_value is not None else ""), - signature=(str(signature_value) if signature_value is not None else None), + signature=(str(signature_value) if signature_value is not None else ""), ).model_dump() ) elif thinking_block.get("type") == "redacted_thinking": @@ -1556,7 +1556,7 @@ class LiteLLMAnthropicMessagesAdapter: AnthropicResponseContentBlockThinking( type="thinking", thinking=str(choice.message.reasoning_content), - signature=None, + signature="", ).model_dump() ) diff --git a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py index 3a26737a8ac..e0256a3c2ae 100644 --- a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py +++ b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py @@ -103,14 +103,12 @@ class FakeAnthropicMessagesStreamIterator: "delta": {"type": "thinking_delta", "thinking": thinking_text}, } chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()) - signature: Final = block_dict.get("signature", "") - if signature: - signature_delta: Final = { - "type": "content_block_delta", - "index": index, - "delta": {"type": "signature_delta", "signature": signature}, - } - chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()) + signature_delta: Final = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "signature_delta", "signature": block_dict.get("signature") or ""}, + } + chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()) elif block_type in ("tool_use", "server_tool_use"): empty_input: Final[Mapping[str, object]] = {} diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index 006cdec9c69..d0f54f28019 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -294,16 +294,11 @@ def _anthropic_content_block_start_and_deltas( ) case "thinking": signature: Final = block.get("signature") - signature_deltas: Final = ( - ({"signature": signature, "type": "signature_delta"},) - if isinstance(signature, str) and signature - else () - ) return ( {"thinking": "", "signature": "", "type": "thinking"}, ( {"thinking": block.get("thinking") or "", "type": "thinking_delta"}, - *signature_deltas, + {"signature": signature if isinstance(signature, str) else "", "type": "signature_delta"}, ), ) case "redacted_thinking": diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index c2c7244259b..6266abeeeb3 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -52,6 +52,7 @@ from ...common_utils import ( requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, + strip_unsignable_thinking_blocks_for_claude, tools_without_custom_field, ) from .mid_conversation_system import ( @@ -779,12 +780,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts ) log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) + base_model: Final = getattr(litellm_params, "base_model", None) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( - drop_non_string_tool_result_texts( - _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, - getattr(litellm_params, "drop_params", None), - ) + messages=strip_unsignable_thinking_blocks_for_claude( + strip_encrypted_reasoning_blocks_from_anthropic_messages( + drop_non_string_tool_result_texts( + _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, + getattr(litellm_params, "drop_params", None), + ) + ), + model=model, + base_model=base_model if isinstance(base_model, str) else None, ), max_tokens=max_tokens, model=model, diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 8bffab03e12..bffbbaf5a81 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -124,6 +124,7 @@ class AnthropicResponsesStreamWrapper: response.output_text.delta -> content_block_delta (text_delta) response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator) response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.reasoning_text.delta -> content_block_delta (thinking_delta) response.function_call_arguments.delta -> content_block_delta (input_json_delta) response.output_item.done -> content_block_delta (signature_delta) + content_block_stop response.completed -> message_delta + message_stop @@ -203,32 +204,52 @@ class AnthropicResponsesStreamWrapper: def _field(source: object, name: str) -> object: return source.get(name) if isinstance(source, dict) else getattr(source, name, None) + def _replay_whole_reasoning_item(self, item: object, item_id: str | None) -> None: + block: Final = LiteLLMAnthropicToResponsesAPIAdapter.thinking_block_from_reasoning_item( + iter_items(self._field(item, "summary") or self._field(item, "content") or ()), + self._field(item, "encrypted_content"), + ) + if block is None: + return + if block.get("type") == "redacted_thinking": + redacted_idx: Final = self._open_block(item_id, block) + self._chunk_queue.append({"type": "content_block_stop", "index": redacted_idx}) + return + block_idx: Final = self._open_block(item_id, {"type": "thinking", "thinking": "", "signature": ""}) + self._chunk_queue.extend( + ( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": block.get("thinking")}, + }, + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": block.get("signature")}, + }, + {"type": "content_block_stop", "index": block_idx}, + ) + ) + def _close_reasoning_item(self, item: object, item_id: str | None) -> None: block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + self._replay_whole_reasoning_item(item, item_id) + return encrypted_content: Final = self._field(item, "encrypted_content") signature: Final = ( encrypted_reasoning_signature(encrypted_content) if isinstance(encrypted_content, str) and encrypted_content else None ) - if block_idx < 0 and signature is None: - return - if block_idx < 0: - redacted_idx: Final = self._open_block( - item_id, - {"type": "redacted_thinking", "data": signature}, - ) - stop: Final[dict[str, object]] = {"type": "content_block_stop", "index": redacted_idx} - self._chunk_queue.append(stop) - return - if signature is not None: - self._chunk_queue.append( - { - "type": "content_block_delta", - "index": block_idx, - "delta": {"type": "signature_delta", "signature": signature}, - } - ) + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": signature or ""}, + } + ) self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) def _process_event(self, event: object) -> None: @@ -332,8 +353,7 @@ class AnthropicResponsesStreamWrapper: ) return - # ---- reasoning summary text delta ---- - if event_type == "response.reasoning_summary_text.delta": + if event_type in ("response.reasoning_summary_text.delta", "response.reasoning_text.delta"): item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index a63884f9afa..f8e49be9f2a 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -8,10 +8,12 @@ path used for OpenAI and Azure models. import json from collections.abc import Iterable, Mapping, Sequence from itertools import groupby +from types import MappingProxyType from typing import Any, Final, cast -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, @@ -65,6 +67,12 @@ RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" _DECODED_JSON: Final = TypeAdapter(object) +def _output_item_mapping(item: object) -> Mapping[str, object]: + if isinstance(item, BaseModel): + return item.model_dump() + return as_str_mapping(item) or MappingProxyType({}) + + class LiteLLMAnthropicToResponsesAPIAdapter: """ Converts Anthropic /v1/messages requests to OpenAI Responses API format and @@ -248,7 +256,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return str(getattr(part, "text", None) or "") @classmethod - def _thinking_block_from_reasoning_item( + def thinking_block_from_reasoning_item( cls, summary: Iterable[object], encrypted_content: object, @@ -266,7 +274,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if not isinstance(encrypted_content, str) or not encrypted_content: if not text: return None - return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump() + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature="").model_dump() signature: Final = encrypted_reasoning_signature(encrypted_content) if not text: return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump() @@ -744,94 +752,52 @@ class LiteLLMAnthropicToResponsesAPIAdapter: """ Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. """ - from openai.types.responses import ( - ResponseFunctionToolCall, - ResponseOutputMessage, - ResponseReasoningItem, - ) - content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" refusal_text: Final = responses_output_refusal_text( cast(Iterable[object], response.output) # cast-ok: output items re-validated per item ) - for item in response.output: - if isinstance(item, ResponseReasoningItem): - reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content) - if reasoning_block is not None: - content.append(reasoning_block) - - elif isinstance(item, ResponseOutputMessage): - for part in item.content: - part_type = getattr(part, "type", None) + for item in map(_output_item_mapping, response.output): + item_type = item.get("type") + if item_type == "message": + for part in filter(None, map(as_str_mapping, iter_items(item.get("content") or ()))): + part_type = part.get("type") if part_type == "output_text": content.append( - AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() + AnthropicResponseContentBlockText( + type="text", text=str(part.get("text") or "") + ).model_dump() ) elif part_type == "refusal": content.append( AnthropicResponseContentBlockText( - type="text", text=getattr(part, "refusal", "") or "" + type="text", text=str(part.get("refusal") or "") ).model_dump() ) - - elif isinstance(item, ResponseFunctionToolCall): + elif item_type == "reasoning": + reasoning_block = self.thinking_block_from_reasoning_item( + iter_items(item.get("summary") or item.get("content") or ()), + item.get("encrypted_content"), + ) + if reasoning_block is not None: + content.append(reasoning_block) + elif item_type == "function_call": try: - input_data = json.loads(item.arguments) if item.arguments else {} - except (json.JSONDecodeError, TypeError): + input_data = _DECODED_JSON.validate_python(json.loads(str(item.get("arguments") or "{}"))) + except json.JSONDecodeError: input_data = {} content.append( - AnthropicResponseContentBlockToolUse( - type="tool_use", - id=item.call_id or item.id or "", - name=item.name, - input=input_data, + AnthropicResponseContentBlockToolUse.model_validate( + { + "type": "tool_use", + "id": item.get("call_id") or item.get("id") or "", + "name": item.get("name") or "", + "input": input_data, + } ).model_dump(exclude_none=True) ) stop_reason = "tool_use" - - elif isinstance(item, dict): - item_type = item.get("type") - if item_type == "message": - for part in item.get("content", ()): - if isinstance(part, dict): - part_type = part.get("type") - if part_type == "output_text": - content.append( - AnthropicResponseContentBlockText( - type="text", text=part.get("text", "") - ).model_dump() - ) - elif part_type == "refusal": - content.append( - AnthropicResponseContentBlockText( - type="text", text=part.get("refusal", "") or "" - ).model_dump() - ) - elif item_type == "reasoning": - reasoning_block = self._thinking_block_from_reasoning_item( - cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json - item.get("encrypted_content"), - ) - if reasoning_block is not None: - content.append(reasoning_block) - elif item_type == "function_call": - try: - input_data = _DECODED_JSON.validate_python(json.loads(item.get("arguments", "{}"))) - except (json.JSONDecodeError, TypeError): - input_data = {} - content.append( - AnthropicResponseContentBlockToolUse.model_validate( - { - "type": "tool_use", - "id": item.get("call_id") or item.get("id", ""), - "name": item.get("name", ""), - "input": input_data, - } - ).model_dump(exclude_none=True) - ) - stop_reason = "tool_use" if response.status == "incomplete": stop_reason = "max_tokens" elif refusal_text is not None: diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index da9820fd73f..0cd72c5f23b 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -18,6 +18,7 @@ use litellm_host::{ protocol::UnobservedTail, }; use litellm_http::Phases; +use litellm_llms::anthropic::messages::transformation::is_claude_model; use litellm_llms::base_llm::{ auth::{Authenticated, AzureAdTokenHosts, ValidatedEnvironment, resolve_auth}, messages::{ @@ -324,6 +325,11 @@ async fn provider_call( let transform_context = MessagesTransformContext { drops_unsupported_params: shaping.settings.drops_unsupported_params, global_drop_params: shaping.settings.global_drop_params, + claude_base_model: shaping + .settings + .base_model + .as_deref() + .is_some_and(is_claude_model), ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) }; let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params); diff --git a/litellm/rust/crates/inference-messages/src/types.rs b/litellm/rust/crates/inference-messages/src/types.rs index 4e9f412df94..3eda7fc0b86 100644 --- a/litellm/rust/crates/inference-messages/src/types.rs +++ b/litellm/rust/crates/inference-messages/src/types.rs @@ -105,6 +105,8 @@ pub struct MessagesSettings { #[serde(skip)] pub module_api_key: Option, #[serde(default)] + pub base_model: Option, + #[serde(default)] pub detached_drain_timeout: Option, #[serde(default)] pub detached_drain_limit: Option, diff --git a/litellm/rust/crates/inference-messages/tests/messages/request.rs b/litellm/rust/crates/inference-messages/tests/messages/request.rs index bc2e108d57c..eb256397cc5 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/request.rs @@ -581,6 +581,50 @@ async fn additional_drop_params_remove_fields_before_sending(call: MessagesCall) assert_eq!(sent["top_k"], 3); } +#[rstest] +#[case::claude_base_model(Some("anthropic/claude-sonnet-4-5"), json!([]))] +#[case::open_base_model(Some("deepseek/deepseek-v4-pro"), json!([{"type": "thinking", "thinking": "plan", "signature": ""}]))] +#[case::no_base_model(None, json!([{"type": "thinking", "thinking": "plan", "signature": ""}]))] +#[tokio::test] +async fn unsigned_thinking_is_dropped_for_an_alias_whose_base_model_names_claude( + call: MessagesCall, + #[case] base_model: Option<&str>, + #[case] sent_thinking: Value, +) { + let upstream = upstream([message_response()]).await; + let history = json!([ + {"role": "user", "content": "plan it"}, + {"role": "assistant", "content": [{"type": "thinking", "thinking": "plan", "signature": ""}]}, + {"role": "user", "content": "go on"} + ]); + + run_message(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + settings: MessagesSettings { + base_model: base_model.map(str::to_owned), + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields(call, json!({"model": "prod-sonnet", "messages": history})) + }) + .await; + + let sent = only_request(&upstream).await.json(); + let thinking: Vec = sent["messages"] + .as_array() + .unwrap() + .iter() + .filter_map(|message| message["content"].as_array()) + .flatten() + .filter(|block| block["type"] == "thinking") + .cloned() + .collect(); + assert_eq!(Value::from(thinking), sent_thinking); +} + fn sent_betas(request: &wiremock::Request) -> BetaSet { let [header] = <[&str; 1]>::try_from(request.header_values("anthropic-beta")) .unwrap_or_else(|values| panic!("expected one anthropic-beta header, got {values:?}")); diff --git a/litellm/rust/crates/llms/src/anthropic/common_utils.rs b/litellm/rust/crates/llms/src/anthropic/common_utils.rs index b5be26e9fb6..0a2bb8783d7 100644 --- a/litellm/rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm/rust/crates/llms/src/anthropic/common_utils.rs @@ -554,6 +554,17 @@ pub fn strip_encrypted_reasoning_blocks(messages: Vec) -> Vec retain_blocks(messages, |block| !is_encrypted_reasoning_block(block)) } +fn is_unsigned_thinking_block(block: &ContentBlock) -> bool { + block.is_type(ContentBlockType::Thinking) + && block.signature.as_deref().is_none_or(str::is_empty) +} + +/// Python's `strip_unsignable_thinking_blocks_for_claude` once the encrypted strip has run: Claude +/// answers 400 to a thinking block with no signature, so an open model's reasoning goes. +pub fn strip_unsigned_thinking_blocks(messages: Vec) -> Vec { + retain_blocks(messages, |block| !is_unsigned_thinking_block(block)) +} + /// Python's `strip_encrypted_reasoning_blocks_from_anthropic_messages`: the same strip, and a /// kept `redacted_thinking` block gets Claude's own `data` back from a Bedrock Converse wrapper. pub fn strip_encrypted_reasoning_blocks_restoring_redacted_data( @@ -1107,6 +1118,25 @@ pub fn strip_raw_encrypted_reasoning_blocks(message: Value) -> Option { }) } +fn is_raw_unsigned_thinking_block(block: &Value) -> bool { + raw_block_type(block) == Some("thinking") + && block + .get("signature") + .and_then(Value::as_str) + .is_none_or(str::is_empty) +} + +pub fn strip_raw_unsigned_thinking_blocks(message: Value) -> Option { + with_raw_blocks(message, |blocks| { + let kept: Vec = blocks + .iter() + .filter(|block| !is_raw_unsigned_thinking_block(block)) + .cloned() + .collect(); + (kept == blocks || !kept.is_empty()).then_some(kept) + }) +} + pub fn is_anthropic_invalid_thinking_block_error(error_text: &str) -> bool { let lower = error_text.to_lowercase(); if !lower.contains("thinking") { @@ -1741,6 +1771,34 @@ mod tests { ); } + #[rstest] + #[case::empty_signature( + json!([{"type": "thinking", "thinking": "plan", "signature": ""}, unparsed_block()]), + Some(json!([unparsed_block()])) + )] + #[case::non_string_signature( + json!([{"type": "thinking", "thinking": "plan", "signature": 5}, unparsed_block()]), + Some(json!([unparsed_block()])) + )] + #[case::signed( + json!([{"type": "thinking", "thinking": "plan", "signature": "sig"}, unparsed_block()]), + Some(json!([{"type": "thinking", "thinking": "plan", "signature": "sig"}, unparsed_block()])) + )] + #[case::redacted_kept( + json!([{"type": "redacted_thinking", "data": "EmwK"}, unparsed_block()]), + Some(json!([{"type": "redacted_thinking", "data": "EmwK"}, unparsed_block()])) + )] + #[case::only_unsigned(json!([{"type": "thinking", "thinking": "plan"}]), None)] + fn strip_raw_unsigned_thinking_blocks_drops_what_claude_rejects( + #[case] blocks: Value, + #[case] expected: Option, + ) { + assert_eq!( + strip_raw_unsigned_thinking_blocks(turn(blocks)), + expected.map(turn) + ); + } + #[rstest] #[case::string_content(json!({"role": "user", "content": 5}))] #[case::not_an_object(json!("hi"))] diff --git a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs index 07cb3d9684f..94fd778a8aa 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs @@ -37,8 +37,9 @@ use crate::{ minted_oauth_environment, optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks_restoring_redacted_data, strip_raw_advisor_blocks, - strip_raw_encrypted_reasoning_blocks, - strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict, + strip_raw_encrypted_reasoning_blocks, strip_raw_unsigned_thinking_blocks, + strip_thinking_blocks_from_anthropic_messages_request, strip_unsigned_thinking_blocks, + tools_without_strict, }, base_llm::{ auth::AuthScheme, @@ -202,7 +203,7 @@ fn is_foundry_url(url: &str) -> bool { }) } -pub(crate) fn is_claude_model(model: &str) -> bool { +pub fn is_claude_model(model: &str) -> bool { model.to_lowercase().contains("claude") } @@ -345,12 +346,22 @@ pub(crate) fn transform_messages_request( } else { messages }; - Ok(MessagesRequest { - messages: map_messages( + let messages = map_messages( + messages, + strip_encrypted_reasoning_blocks_restoring_redacted_data, + strip_raw_encrypted_reasoning_blocks, + ); + let messages = if context.claude_base_model || is_claude_model(&request.model) { + map_messages( messages, - strip_encrypted_reasoning_blocks_restoring_redacted_data, - strip_raw_encrypted_reasoning_blocks, - ), + strip_unsigned_thinking_blocks, + strip_raw_unsigned_thinking_blocks, + ) + } else { + messages + }; + Ok(MessagesRequest { + messages, params: MessagesOptionalParams { context_management, extra: without_null_safeguards(request.params.extra), @@ -1317,6 +1328,58 @@ mod tests { ); } + fn tool_turn(opening: Option) -> Value { + let pending: Vec = opening + .into_iter() + .chain([json!({"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}})]) + .collect(); + json!([ + {"role": "user", "content": "List the files."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "claude plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "text", "text": "Draft."} + ]}, + {"role": "user", "content": "Now list them."}, + {"role": "assistant", "content": pending}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]} + ]) + } + + #[rstest] + #[case::empty(json!({"type": "thinking", "thinking": "open model plan", "signature": ""}))] + #[case::null(json!({"type": "thinking", "thinking": "open model plan", "signature": null}))] + #[case::missing(json!({"type": "thinking", "thinking": "open model plan"}))] + fn unsigned_thinking_never_reaches_claude_even_on_the_pending_tool_turn( + #[case] unsigned: Value, + unmapped: MessagesModelCapabilities, + ) { + assert_eq!( + transform( + json!({"messages": tool_turn(Some(unsigned))}), + unmapped, + false + ), + Ok(body(json!({"messages": tool_turn(None)}))) + ); + } + + #[rstest] + fn unsigned_thinking_is_still_replayed_to_an_open_model(unmapped: MessagesModelCapabilities) { + let messages = tool_turn(Some( + json!({"type": "thinking", "thinking": "open model plan", "signature": ""}), + )); + assert_eq!( + transform( + json!({"model": "deepseek-v4-pro", "messages": messages.clone()}), + unmapped, + false + ), + Ok(body( + json!({"model": "deepseek-v4-pro", "messages": messages}) + )) + ); + } + #[test] fn thinking_is_translated_with_the_context_budgets() { let context = MessagesTransformContext::with_lookup( @@ -1360,6 +1423,7 @@ mod tests { drop_params: true, drops_unsupported_params: false, global_drop_params: false, + claude_base_model: false, } ); return; diff --git a/litellm/rust/crates/llms/src/base_llm/messages/context.rs b/litellm/rust/crates/llms/src/base_llm/messages/context.rs index 706821863f3..fdd3aae42f7 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/context.rs @@ -178,6 +178,7 @@ pub struct MessagesTransformContext { pub drop_params: bool, pub drops_unsupported_params: bool, pub global_drop_params: bool, + pub claude_base_model: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -199,6 +200,7 @@ impl MessagesTransformContext { drop_params, drops_unsupported_params: false, global_drop_params: false, + claude_base_model: false, } } diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs index a63f1ccb4d4..dfc7d28826a 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs @@ -696,6 +696,7 @@ impl MessagesPythonHost { unencodable_request_header, module_api_key: azure.as_ref().and_then(|azure| azure.api_key.clone()), global_drop_params: global_drop_params(py)?, + base_model: argument("base_model")?.and_then(|value| value.extract().ok()), ..settings.settings }, }; diff --git a/litellm/rust/crates/router/src/lib.rs b/litellm/rust/crates/router/src/lib.rs index b7440cffc7f..6b1cc30e27d 100644 --- a/litellm/rust/crates/router/src/lib.rs +++ b/litellm/rust/crates/router/src/lib.rs @@ -65,6 +65,11 @@ impl Router { additional_drop_params: additional_drop_params( &litellm_params.extra, ), + base_model: litellm_params + .extra + .get("base_model") + .and_then(|value| value.as_str()) + .map(str::to_owned), ..MessagesSettings::default() }, capabilities: anthropic_model_capabilities( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 244883859f0..60e38650131 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -747,7 +747,7 @@ class AnthropicResponseContentBlockToolUse(LiteLLMBaseModel): class AnthropicResponseContentBlockThinking(LiteLLMBaseModel): type: Literal["thinking"] thinking: str - signature: str | None + signature: str class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel): diff --git a/tests/integration/streaming/test_stream_contracts.py b/tests/integration/streaming/test_stream_contracts.py index 1d2a8b25792..5e02892de30 100644 --- a/tests/integration/streaming/test_stream_contracts.py +++ b/tests/integration/streaming/test_stream_contracts.py @@ -447,6 +447,7 @@ def test_messages_stream_opens_thinking_block_at_index_zero_for_reasoning_conten (0, {"type": "thinking", "thinking": "", "signature": ""}), (0, {"type": "thinking_delta", "thinking": "Let me "}), (0, {"type": "thinking_delta", "thinking": "think."}), + (0, {"type": "signature_delta", "signature": ""}), (1, {"type": "text", "text": ""}), (1, {"type": "text_delta", "text": "Hello "}), (1, {"type": "text_delta", "text": "there"}), @@ -456,6 +457,7 @@ def test_messages_stream_opens_thinking_block_at_index_zero_for_reasoning_conten "content_block_start", "content_block_delta", "content_block_delta", + "content_block_delta", "content_block_stop", "content_block_start", "content_block_delta", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index a76d5e82a37..eec1c33e217 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -23,6 +23,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) +from litellm.llms.anthropic.messages.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, +) from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, @@ -3233,7 +3236,7 @@ def test_translate_openai_content_to_anthropic_reasoning_content_without_thinkin assert result[0]["type"] == "thinking" assert "Considering Letter Frequency" in result[0]["thinking"] assert "Calculating the Count" in result[0]["thinking"] - assert result[0]["signature"] is None + assert result[0]["signature"] == "" # Second block should be text block with content assert result[1]["type"] == "text" assert result[1]["text"] == 'There are **3** "r"s in the word strawberry.' @@ -3311,7 +3314,7 @@ def test_translate_openai_response_to_anthropic_with_reasoning_content_only(): # First block should be thinking assert anthropic_content[0]["type"] == "thinking" assert "Considering Letter Frequency" in anthropic_content[0]["thinking"] - assert anthropic_content[0].get("signature") is None + assert anthropic_content[0].get("signature") == "" # Second block should be text assert anthropic_content[1]["type"] == "text" @@ -6293,3 +6296,71 @@ async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item( ("thinking", "Check the forecast.", encrypted_reasoning_signature("gAAA_second")), ("text", "", None), ] + + +_TOOL_LOOP_FROM_AN_OPEN_MODEL: Final = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "open model plan", "signature": ""}, + {"type": "text", "text": "Draft."}, + ], + }, + {"role": "user", "content": "List the files."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "call ls", "signature": ""}, + {"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]}, +] + + +def _sent_assistant_turns(model: str, deployment: Mapping[str, object]) -> list[dict[str, object]]: + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=_TOOL_LOOP_FROM_AN_OPEN_MODEL, + model=model, + thinking={"type": "enabled", "budget_tokens": 1024}, + extra_kwargs=deployment, + ) + return [m for m in completion_kwargs["messages"] if m["role"] == "assistant"] + + +@pytest.mark.parametrize( + "model, deployment", + [ + pytest.param( + "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + {"custom_llm_provider": "bedrock"}, + id="bedrock-converse", + ), + pytest.param( + "github_copilot/claude-sonnet-4.5", + {"custom_llm_provider": "github_copilot"}, + id="github-copilot", + ), + pytest.param( + "bedrock/converse/arn:aws:bedrock:us-west-2:123456789012:application-inference-profile/abc123", + {"custom_llm_provider": "bedrock", "base_model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"}, + id="application-inference-profile-with-base-model", + ), + ], +) +def test_unsigned_thinking_never_reaches_claude_through_the_chat_adapter( + model: str, deployment: Mapping[str, object] +) -> None: + turns: Final = _sent_assistant_turns(model, deployment) + + assert [turn.get("thinking_blocks") for turn in turns] == [None, None] + assert [turn.get("reasoning_content") for turn in turns] == [None, None] + assert turns[1]["tool_calls"][0]["id"] == "toolu_1" + + +def test_unsigned_thinking_is_still_replayed_to_an_open_model_through_the_chat_adapter() -> None: + turns: Final = _sent_assistant_turns("deepseek/deepseek-v4-pro", {"custom_llm_provider": "deepseek"}) + + assert [turn.get("reasoning_content") for turn in turns] == ["open model plan", "call ls"] diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py index 79cde87b4e0..28c283205ea 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py @@ -1534,3 +1534,56 @@ async def test_signed_thinking_blocks_keep_their_own_text_and_signature(is_async assert _last_delta_types_per_thinking_block(events) == ("signature_delta", "signature_delta") assert _text_deltas(events) == ["Done"] _assert_deltas_match_their_block_type(events) + + +_UNSIGNED_REASONING_STREAMS: Final[Mapping[str, Tuple[MagicMock, ...]]] = MappingProxyType( + { + "then-text": ( + _reasoning_content_chunk("The user "), + _reasoning_content_chunk("wants 2+2."), + _make_chunk(Delta(content="4")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ), + "then-tool-call": ( + _reasoning_content_chunk("The user wants 2+2."), + _tool_chunk("call_add", "add", '{"a": 2, "b": 2}'), + _make_chunk(Delta(content=None), finish_reason="tool_calls"), + ), + "thought-ends-with-finish": ( + _reasoning_content_chunk("The user "), + _make_chunk(Delta(content=None, reasoning_content="wants 2+2."), finish_reason="stop"), + ), + "unsigned-thinking-blocks": ( + _thinking_chunk("The user "), + _thinking_chunk("wants 2+2."), + _make_chunk(Delta(content="4")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ), + } +) + + +def _event_before_stop(events: Sequence[Mapping[str, object]], index: int) -> Mapping[str, object]: + stop_position: Final = next( + position + for position, event in enumerate(events) + if event.get("type") == "content_block_stop" and event["index"] == index + ) + return events[stop_position - 1] + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("stream", list(_UNSIGNED_REASONING_STREAMS), ids=list(_UNSIGNED_REASONING_STREAMS)) +@pytest.mark.asyncio +async def test_unsigned_thinking_block_closes_with_an_empty_signature_delta(is_async: bool, stream: str): + events: Final = await _events_for(_UNSIGNED_REASONING_STREAMS[stream], is_async) + + assert _thinking_blocks_as_accumulated(events) == ( + _AccumulatedThinking(thinking="The user wants 2+2.", signature=""), + ) + assert [_event_before_stop(events, index) for index in _thinking_starts(events)] == [ + {"type": "content_block_delta", "index": index, "delta": {"type": "signature_delta", "signature": ""}} + for index in _thinking_starts(events) + ] + assert _signature_deltas(events) == [""] + _assert_deltas_match_their_block_type(events) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py index 4b52f7fbef1..703fae5e614 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py @@ -1,9 +1,15 @@ +from typing import Final + +import pytest + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) +from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig +from litellm.types.router import GenericLiteLLMParams def _transform(messages): @@ -48,3 +54,126 @@ def test_anthropic_signed_thinking_blocks_are_forwarded_untouched(): }, ] assert _transform(messages)["messages"] == messages + + +_CLAUDE_SIGNED_TURN: Final = { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "claude plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "text", "text": "Refined."}, + ], +} + + +def _open_model_turn(thinking_block: dict[str, object]) -> dict[str, object]: + return {"role": "assistant", "content": [thinking_block, {"type": "text", "text": "Draft."}]} + + +@pytest.mark.parametrize( + "thinking_block", + [ + {"type": "thinking", "thinking": "open model plan", "signature": ""}, + {"type": "thinking", "thinking": "open model plan", "signature": None}, + {"type": "thinking", "thinking": "open model plan"}, + ], + ids=["empty", "null", "missing"], +) +def test_unsigned_thinking_from_an_open_model_never_reaches_claude(thinking_block: dict[str, object]) -> None: + messages: Final = [ + {"role": "user", "content": "Solve it."}, + _open_model_turn(thinking_block), + {"role": "user", "content": "Refine it."}, + _CLAUDE_SIGNED_TURN, + {"role": "user", "content": "Thanks."}, + ] + + sent: Final = _transform(messages)["messages"] + + assert sent[1]["content"] == [{"type": "text", "text": "Draft."}] + assert sent[3] == _CLAUDE_SIGNED_TURN + assert messages[1]["content"][0] == thinking_block + + +def test_unsigned_thinking_is_still_replayed_to_an_open_model() -> None: + messages: Final = [ + {"role": "user", "content": "Solve it."}, + _open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}), + {"role": "user", "content": "Refine it."}, + ] + + sent: Final = DeepSeekAnthropicMessagesConfig().transform_anthropic_messages_request( + model="deepseek-v4-pro", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params={}, + headers={}, + )["messages"] + + assert sent[1]["content"][0]["thinking"] == "open model plan" + + +def _pending_tool_turn(opening: list[dict[str, object]]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "List the files."}, + { + "role": "assistant", + "content": [*opening, {"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "a.txt"}]}, + ] + + +def test_unsigned_thinking_that_opens_the_pending_tool_turn_is_dropped_too() -> None: + messages: Final = [ + {"role": "user", "content": "Solve it."}, + _open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}), + *_pending_tool_turn([{"type": "thinking", "thinking": "call ls", "signature": ""}]), + ] + + sent: Final = _transform(messages)["messages"] + + assert sent[1]["content"] == [{"type": "text", "text": "Draft."}] + assert sent[3]["content"] == [{"type": "tool_use", "id": "toolu_1", "name": "ls", "input": {}}] + assert sent[4] == messages[4] + + +def test_unsigned_thinking_is_dropped_when_the_pending_tool_turn_never_opened_with_thinking() -> None: + messages: Final = [ + {"role": "user", "content": "Solve it."}, + _open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}), + *_pending_tool_turn([]), + ] + + sent: Final = _transform(messages)["messages"] + + assert sent[1]["content"] == [{"type": "text", "text": "Draft."}] + assert sent[2:] == messages[2:] + + +@pytest.mark.parametrize( + ("base_model", "sent_opening"), + [ + ("anthropic/claude-sonnet-4-5", {"type": "text", "text": "Draft."}), + ("deepseek/deepseek-v4-pro", {"type": "thinking", "thinking": "open model plan", "signature": ""}), + (None, {"type": "thinking", "thinking": "open model plan", "signature": ""}), + ], + ids=["claude-base-model", "open-base-model", "no-base-model"], +) +def test_an_alias_strips_unsigned_thinking_only_when_its_base_model_names_claude( + base_model: str | None, sent_opening: dict[str, object] +) -> None: + messages: Final = [ + {"role": "user", "content": "Solve it."}, + _open_model_turn({"type": "thinking", "thinking": "open model plan", "signature": ""}), + {"role": "user", "content": "Refine it."}, + ] + + sent: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="prod-sonnet", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params=GenericLiteLLMParams(base_model=base_model), + headers={}, + )["messages"] + + assert sent[1]["content"][0] == sent_opening diff --git a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py index ece5bb39225..69e68b43e7c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py @@ -232,3 +232,32 @@ def _sdk_rebuilt_content(block: Mapping[str, object]) -> list[dict[str, object]] @pytest.mark.parametrize("block", _REPLAYED_BLOCKS, ids=lambda block: str(block["type"])) def test_anthropic_sdk_rebuilds_every_block_field_from_the_stream(block: Mapping[str, object]): assert _sdk_rebuilt_content(block) == [block] + + +@pytest.mark.parametrize( + "unsigned_block", + [ + {"type": "thinking", "thinking": "Let me think.", "signature": ""}, + {"type": "thinking", "thinking": "Let me think.", "signature": None}, + {"type": "thinking", "thinking": "Let me think."}, + ], + ids=["empty", "null", "missing"], +) +def test_unsigned_thinking_block_closes_with_an_empty_signature_delta(unsigned_block: Mapping[str, object]): + response: Final = { + "id": "msg_unsigned", + "content": [unsigned_block, {"type": "text", "text": "Hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + thinking_events: Final = tuple( + (event_type, payload.get("delta", payload.get("content_block"))) + for event_type, payload in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response)) + if payload.get("index") == 0 + ) + assert thinking_events == ( + ("content_block_start", {"type": "thinking", "thinking": "", "signature": ""}), + ("content_block_delta", {"type": "thinking_delta", "thinking": "Let me think."}), + ("content_block_delta", {"type": "signature_delta", "signature": ""}), + ("content_block_stop", None), + ) diff --git a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index 59a4a882624..0dfbf5acac2 100644 --- a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -1372,15 +1372,25 @@ def test_anthropic_messages_response_as_sse_events_thinking_block_emits_signatur ] -def test_anthropic_messages_response_as_sse_events_thinking_block_without_signature_omits_delta(): - response = { - "id": "msg_6", - "content": [{"type": "thinking", "thinking": "let me think", "signature": None}], - "stop_reason": "end_turn", - } +@pytest.mark.parametrize( + "thinking_block", + [ + {"type": "thinking", "thinking": "let me think", "signature": None}, + {"type": "thinking", "thinking": "let me think", "signature": ""}, + {"type": "thinking", "thinking": "let me think"}, + ], + ids=["null", "empty", "missing"], +) +def test_anthropic_messages_response_as_sse_events_unsigned_thinking_block_closes_with_an_empty_signature( + thinking_block, +): + response = {"id": "msg_6", "content": [thinking_block], "stop_reason": "end_turn"} decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response)) deltas = [payload["delta"] for event_type, payload in decoded if event_type == "content_block_delta"] - assert deltas == [{"type": "thinking_delta", "thinking": "let me think"}] + assert deltas == [ + {"type": "thinking_delta", "thinking": "let me think"}, + {"type": "signature_delta", "signature": ""}, + ] def test_anthropic_messages_response_as_sse_events_multiple_blocks_are_indexed(): diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index 1e22ce1fc97..9faff1dbf1b 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -179,6 +179,7 @@ class TestReasoningItemWithoutSummaryText: ("content_block_start", 0), ("content_block_delta", 0), ("content_block_delta", 0), + ("content_block_delta", 0), ("content_block_stop", 0), ("content_block_start", 1), ("content_block_delta", 1), @@ -190,10 +191,73 @@ class TestReasoningItemWithoutSummaryText: assert "".join(c["delta"]["thinking"] for c in chunks[2:4]) == "Weighing options" def test_the_reasoning_item_id_is_never_streamed_as_a_signature(self): - """A stand-in signature would be replayed as a real one, so none is ever sent.""" + """A stand-in signature would be replayed as a real one, so the block closes with an empty one.""" chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["Weighing options"])) - assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] + signature_deltas: Final = [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] + assert [(c["index"], c["delta"]["signature"]) for c in signature_deltas] == [(0, "")] + assert chunks[chunks.index(signature_deltas[0]) + 1] == {"type": "content_block_stop", "index": 0} + + +class TestRawReasoningTextIsStreamedAsThinking: + """Servers that return a reasoning item's raw text (vLLM and OpenRouter for gpt-oss) + stream it as ``response.reasoning_text.delta`` instead of a summary.""" + + def test_reasoning_text_deltas_become_one_closed_thinking_block(self): + chunks: Final = _drain_async( + [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.reasoning_text.delta", "item_id": "rs_1", "content_index": 0, "delta": "17*23"}, + {"type": "response.reasoning_text.delta", "item_id": "rs_1", "content_index": 0, "delta": " is 391."}, + {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "391"}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.completed"}, + ] + ) + + assert [(c["type"], c.get("index"), c.get("content_block") or c.get("delta")) for c in chunks[1:-2]] == [ + ("content_block_start", 0, {"type": "thinking", "thinking": "", "signature": ""}), + ("content_block_delta", 0, {"type": "thinking_delta", "thinking": "17*23"}), + ("content_block_delta", 0, {"type": "thinking_delta", "thinking": " is 391."}), + ("content_block_delta", 0, {"type": "signature_delta", "signature": ""}), + ("content_block_stop", 0, None), + ("content_block_start", 1, {"type": "text", "text": ""}), + ("content_block_delta", 1, {"type": "text_delta", "text": "391"}), + ("content_block_stop", 1, None), + ] + + def test_a_whole_reasoning_item_without_deltas_becomes_one_closed_thinking_block(self): + reasoning_item: Final = { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "content": [{"type": "reasoning_text", "text": "Just answer 391."}], + "encrypted_content": None, + } + chunks: Final = _drain_async( + [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": reasoning_item}, + {"type": "response.output_item.done", "item": reasoning_item}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "391"}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.completed"}, + ] + ) + + assert [(c["type"], c.get("index"), c.get("content_block") or c.get("delta")) for c in chunks[1:-2]] == [ + ("content_block_start", 0, {"type": "thinking", "thinking": "", "signature": ""}), + ("content_block_delta", 0, {"type": "thinking_delta", "thinking": "Just answer 391."}), + ("content_block_delta", 0, {"type": "signature_delta", "signature": ""}), + ("content_block_stop", 0, None), + ("content_block_start", 1, {"type": "text", "text": ""}), + ("content_block_delta", 1, {"type": "text_delta", "text": "391"}), + ("content_block_stop", 1, None), + ] _ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 7ef69d4c791..7c64acccb14 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -11,7 +11,14 @@ from typing import Any, Dict, Final, List, Literal from unittest.mock import MagicMock import pytest -from openai.types.responses import ResponseInputImageParam +from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseInputImageParam, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, +) +from openai.types.responses.response_reasoning_item import Summary from pydantic import TypeAdapter from litellm.constants import ( @@ -33,11 +40,15 @@ from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthropicMessagesRequest, ) -from litellm.types.llms.openai import ResponseAPIUsage +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import Choices, Message, ModelResponse def _make_request(**overrides) -> AnthropicMessagesRequest: @@ -1410,20 +1421,14 @@ def _make_mock_response( return resp -def _make_output_message(texts: List[str]) -> MagicMock: - """Build a mock ResponseOutputMessage with output_text parts.""" - from openai.types.responses import ResponseOutputMessage # type: ignore[import] - - parts = [] - for t in texts: - part = MagicMock() - part.type = "output_text" - part.text = t - parts.append(part) - - msg = MagicMock(spec=ResponseOutputMessage) - msg.content = parts - return msg +def _make_output_message(texts: List[str]) -> ResponseOutputMessage: + return ResponseOutputMessage( + id="msg_test_1", + content=[ResponseOutputText(type="output_text", text=text, annotations=[]) for text in texts], + role="assistant", + status="completed", + type="message", + ) def _make_refusal_message(refusal_text: str): @@ -1438,35 +1443,19 @@ def _make_refusal_message(refusal_text: str): ) -def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: - """Build a mock ResponseFunctionToolCall.""" - from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] - - item = MagicMock(spec=ResponseFunctionToolCall) - item.call_id = call_id - item.id = call_id - item.name = name - item.arguments = arguments - return item +def _make_function_call_item(call_id: str, name: str, arguments: str) -> ResponseFunctionToolCall: + return ResponseFunctionToolCall(call_id=call_id, id=call_id, name=name, arguments=arguments, type="function_call") def _make_reasoning_item( summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None -) -> MagicMock: - """Build a mock ResponseReasoningItem.""" - from openai.types.responses import ResponseReasoningItem # type: ignore[import] - - summary_mocks = [] - for text in summaries: - s = MagicMock() - s.text = text - summary_mocks.append(s) - - item = MagicMock(spec=ResponseReasoningItem) - item.id = item_id - item.summary = summary_mocks - item.encrypted_content = encrypted_content - return item +) -> ResponseReasoningItem: + return ResponseReasoningItem( + id=item_id, + summary=[Summary(text=text, type="summary_text") for text in summaries], + type="reasoning", + encrypted_content=encrypted_content, + ) _ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" @@ -1596,7 +1585,7 @@ class TestTranslateResponse: reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") response = _make_mock_response(output=[reasoning]) result: Any = _ADAPTER.translate_response(response) - assert [block["signature"] for block in result["content"]] == [None] + assert [block["signature"] for block in result["content"]] == [""] assert "rs_abc123" not in json.dumps(result["content"]) def test_summary_parts_join_into_one_thinking_block(self): @@ -1684,7 +1673,7 @@ class TestTranslateResponse: ] ) result: Any = _ADAPTER.translate_response(response) - assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] + assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": ""}] @pytest.mark.parametrize( ("summaries", "encrypted_content"), @@ -2530,3 +2519,51 @@ class TestPromptCacheBreakpointToResponses: extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, ) assert kwargs["prompt_cache_options"] == {"mode": "explicit"} + + +class TestTranslateChatBridgedResponse: + @staticmethod + def _bridged(message: Message, finish_reason: str) -> ResponsesAPIResponse: + return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Say OK", + responses_api_request={}, + chat_completion_response=ModelResponse( + model="deepseek-v4-pro", choices=[Choices(index=0, finish_reason=finish_reason, message=message)] + ), + ) + + def test_reasoning_and_text_are_kept(self): + response: Final = self._bridged( + Message(role="assistant", content="OK", reasoning_content="The user wants OK."), "stop" + ) + + result: Final = _ADAPTER.translate_response(response) + + assert result["content"] == [ + {"type": "thinking", "thinking": "The user wants OK.", "signature": ""}, + {"type": "text", "text": "OK"}, + ] + assert result["stop_reason"] == "end_turn" + + def test_tool_call_is_kept(self): + response: Final = self._bridged( + Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Oslo"}'}, + } + ], + ), + "tool_calls", + ) + + result: Final = _ADAPTER.translate_response(response) + + assert result["content"] == [ + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}} + ] + assert result["stop_reason"] == "tool_use" diff --git a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py index ae54d0e7437..179906879ec 100644 --- a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py +++ b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py @@ -265,6 +265,29 @@ def test_request_transform_keeps_the_billing_header_block() -> None: assert request["system"] == [billing_block, {"type": "text", "text": "be brief"}] + +def test_request_transform_drops_thinking_claude_cannot_verify() -> None: + request: Final = DatabricksAnthropicMessagesConfig().transform_anthropic_messages_request( + model=CLAUDE_MODEL, + messages=[ + {"role": "user", "content": "first"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "open model plan", "signature": ""}, + {"type": "text", "text": "reply"}, + ], + }, + {"role": "user", "content": "second"}, + ], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request["messages"][1]["content"] == [{"type": "text", "text": "reply"}] + + def _reply(request: httpx.Request, body: dict[str, object]) -> httpx.Response: return httpx.Response( status_code=200, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_anthropic_streaming_block.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_anthropic_streaming_block.py index 865b4164e5e..28a06cde6c6 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_anthropic_streaming_block.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_anthropic_streaming_block.py @@ -360,3 +360,47 @@ class TestContentBlockState: ] assert BLOCK_MESSAGE in raw assert '"index": 1' in raw + + @pytest.mark.parametrize( + "thinking_so_far, closing_deltas", + [ + ((), [{"type": "signature_delta", "signature": ""}]), + ( + ({"type": "signature_delta", "signature": "sig-abc"},), + [], + ), + ], + ids=["unsigned", "already-signed"], + ) + def test_continuation_signs_an_open_thinking_block_once(self, thinking_so_far, closing_deltas): + from litellm.integrations.custom_guardrail import ModifyResponseException + + seen = [ + _sse_event( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ), + _sse_event( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "Hm"}}, + ), + *( + _sse_event("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": delta}) + for delta in thinking_so_far + ), + ] + exc = ModifyResponseException( + message=BLOCK_MESSAGE, model="claude-3-5-sonnet", request_data={}, guardrail_name="g" + ) + + raw = b"".join( + self._handler().build_block_sse_chunks(exc, stream_started=True, responses_so_far=seen) + ).decode() + payloads = _parse_sse_payloads(raw) + stop_position = payloads.index({"type": "content_block_stop", "index": 0}) + + assert [payload["delta"] for payload in payloads[:stop_position]] == closing_deltas