mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(chatgpt): round-trip encrypted reasoning through /v1/messages
The chat bridge stored encrypted reasoning on reasoning_items, the Anthropic adapter never read that field, stream chunks that only carried thinking blocks were dropped, and the messages adapter stripped the signed blocks before a chatgpt request could turn them back into reasoning items. Co-authored-by: DDT <1786035110@qq.com>
This commit is contained in:
parent
4086d5ccaa
commit
ea9e29b1a8
9 changed files with 581 additions and 36 deletions
|
|
@ -39,7 +39,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
|||
parse_chat_allowed_tools_tool_choice,
|
||||
responses_allowed_tools_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
prompt_cache_breakpoint_with_readable_mode,
|
||||
|
|
@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
thinking_blocks_from_reasoning_items,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
|
|
@ -68,6 +69,8 @@ from litellm.responses.utils import (
|
|||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
|
|
@ -76,7 +79,7 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
|
||||
from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
|
||||
|
|
@ -481,6 +484,63 @@ def _as_chat_reasoning_items(
|
|||
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
|
||||
|
||||
|
||||
def _with_stream_block_index(
|
||||
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
|
||||
index: int,
|
||||
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
|
||||
if block["type"] == "redacted_thinking":
|
||||
data: Final = block.get("data")
|
||||
return ChatCompletionRedactedThinkingBlock(
|
||||
type="redacted_thinking",
|
||||
data=data if isinstance(data, str) else "",
|
||||
index=index,
|
||||
)
|
||||
thinking: Final = block.get("thinking")
|
||||
signature: Final = block.get("signature")
|
||||
return ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
thinking=thinking if isinstance(thinking, str) else "",
|
||||
signature=signature if isinstance(signature, str) else None,
|
||||
index=index,
|
||||
)
|
||||
|
||||
|
||||
def _reasoning_done_thinking_blocks(
|
||||
item: Mapping[str, object],
|
||||
output_index: object,
|
||||
) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]:
|
||||
index: Final = output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else 0
|
||||
return tuple(_with_stream_block_index(block, index) for block in thinking_blocks_from_reasoning_items((item,)))
|
||||
|
||||
|
||||
def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseStream | None:
|
||||
event_type: Final = chunk.get("type")
|
||||
event_name: Final = (
|
||||
event_type.value
|
||||
if isinstance(event_type, ResponsesAPIStreamEvents)
|
||||
else event_type
|
||||
if isinstance(event_type, str)
|
||||
else None
|
||||
)
|
||||
if event_name != ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
|
||||
return None
|
||||
item: Final = as_str_mapping(chunk.get("item"))
|
||||
if item is None or item.get("type") != "reasoning":
|
||||
return None
|
||||
signed_blocks: Final = _reasoning_done_thinking_blocks(item, chunk.get("output_index"))
|
||||
if not signed_blocks:
|
||||
return None
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(thinking_blocks=list(signed_blocks)),
|
||||
finish_reason=None,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
|
||||
shifted: Final = {
|
||||
key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
|
||||
|
|
@ -535,6 +595,7 @@ def _fold_choices_into_one(
|
|||
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
|
||||
*trailing_reasoning_items,
|
||||
)
|
||||
encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items)
|
||||
return Choices(
|
||||
message=Message(
|
||||
role=messages[0].role if messages else "assistant",
|
||||
|
|
@ -543,6 +604,7 @@ def _fold_choices_into_one(
|
|||
tool_calls=list(tool_calls) or None,
|
||||
reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
|
||||
reasoning_items=_as_chat_reasoning_items(reasoning_items),
|
||||
thinking_blocks=list(encrypted_thinking_blocks) or None,
|
||||
provider_specific_fields=_tool_search_provider_fields(tool_search_items),
|
||||
),
|
||||
finish_reason="tool_calls" if tool_calls else "stop",
|
||||
|
|
@ -1436,10 +1498,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
for summary_block in reasoning_item["summary"]
|
||||
if summary_block.get("text")
|
||||
)
|
||||
encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items)
|
||||
message: Final = Message(
|
||||
content=None if tool_search_items else "",
|
||||
reasoning_content=reasoning_content if reasoning_content else None,
|
||||
reasoning_items=_as_chat_reasoning_items(reasoning_items),
|
||||
thinking_blocks=list(encrypted_thinking_blocks) or None,
|
||||
provider_specific_fields=_tool_search_provider_fields(tool_search_items),
|
||||
)
|
||||
return Choices(message=message, finish_reason=finish_reason, index=0)
|
||||
|
|
@ -2248,7 +2312,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
]
|
||||
)
|
||||
|
||||
elif event_type == "response.output_text.delta":
|
||||
# Content part added to output
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
|
|
@ -2365,9 +2428,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
"""
|
||||
verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk)
|
||||
self._remember_served_service_tier(chunk)
|
||||
reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk)
|
||||
translated: Final = (
|
||||
self._empty_content_chunk()
|
||||
if self._is_text_delta_after_structured_answer(chunk)
|
||||
else reasoning_chunk
|
||||
if reasoning_chunk is not None
|
||||
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
|
||||
chunk,
|
||||
tool_call_index_map=self._tool_call_index_map,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
|||
from itertools import groupby
|
||||
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionReasoningSummaryTextBlock,
|
||||
|
|
@ -94,6 +95,87 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
|
|||
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
|
||||
|
||||
|
||||
_REASONING_SUMMARY_SEPARATOR: Final = "\n\n"
|
||||
|
||||
|
||||
def _object_sequence(value: object) -> tuple[object, ...]:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||
return ()
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(mapping for item in _object_sequence(value) if (mapping := as_str_mapping(item)) is not None)
|
||||
|
||||
|
||||
def _reasoning_summary_part_text(part: object) -> str:
|
||||
mapping: Final = as_str_mapping(part)
|
||||
if mapping is not None:
|
||||
text: Final = mapping.get("text")
|
||||
return text if isinstance(text, str) else ""
|
||||
text_attr: Final = getattr(part, "text", None)
|
||||
return text_attr if isinstance(text_attr, str) else ""
|
||||
|
||||
|
||||
def _reasoning_item_summary_text(item: Mapping[str, object]) -> str:
|
||||
parts: Final = tuple(_reasoning_summary_part_text(part) for part in _object_sequence(item.get("summary")))
|
||||
return _REASONING_SUMMARY_SEPARATOR.join(part for part in parts if part)
|
||||
|
||||
|
||||
def thinking_block_from_reasoning_item(
|
||||
item: object,
|
||||
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None:
|
||||
"""The chat thinking block that carries one Responses `reasoning` item back to an Anthropic client.
|
||||
|
||||
`encrypted_content` rides the same `litellm_encrypted_reasoning:` signature the Responses
|
||||
adapter uses. A summary becomes thinking text. Bytes with no summary become `redacted_thinking`,
|
||||
because an empty thinking string is not something the client can replay.
|
||||
"""
|
||||
mapping: Final = as_str_mapping(item)
|
||||
if mapping is None or mapping.get("type") not in (None, "reasoning"):
|
||||
return None
|
||||
encrypted_content: Final = mapping.get("encrypted_content")
|
||||
if not isinstance(encrypted_content, str) or not encrypted_content:
|
||||
return None
|
||||
signature: Final = encrypted_reasoning_signature(encrypted_content)
|
||||
text: Final = _reasoning_item_summary_text(mapping)
|
||||
if not text:
|
||||
return ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=signature)
|
||||
return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature)
|
||||
|
||||
|
||||
def thinking_blocks_from_reasoning_items(
|
||||
items: object,
|
||||
) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]:
|
||||
return tuple(
|
||||
block for item in _object_sequence(items) if (block := thinking_block_from_reasoning_item(item)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _block_encrypted_content(block: object) -> str | None:
|
||||
mapping: Final = as_str_mapping(block)
|
||||
if mapping is None:
|
||||
return None
|
||||
return encrypted_content_of_block(mapping)
|
||||
|
||||
|
||||
def thinking_blocks_with_encrypted_reasoning(
|
||||
thinking_blocks: object,
|
||||
reasoning_items: object,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""Thinking blocks already on the message, plus any encrypted reasoning item they do not already carry."""
|
||||
direct: Final = _iter_mappings(thinking_blocks)
|
||||
represented: Final = frozenset(
|
||||
content for block in direct if (content := _block_encrypted_content(block)) is not None
|
||||
)
|
||||
extra: Final = tuple(
|
||||
block
|
||||
for block in thinking_blocks_from_reasoning_items(_iter_mappings(reasoning_items))
|
||||
if _block_encrypted_content(block) not in represented
|
||||
)
|
||||
return (*extra, *direct)
|
||||
|
||||
|
||||
def is_encrypted_reasoning_block(block: object) -> bool:
|
||||
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
|
||||
|
||||
|
|
|
|||
|
|
@ -1129,6 +1129,7 @@ class CustomStreamWrapper:
|
|||
and model_response.choices[0].delta.role is not None
|
||||
)
|
||||
or (getattr(model_response.choices[0].delta, "reasoning_items", None) is not None)
|
||||
or getattr(model_response.choices[0].delta, "thinking_blocks", None)
|
||||
):
|
||||
return True
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -27,8 +27,9 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, attribute_or_none
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import thinking_blocks_from_reasoning_items
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -92,6 +93,10 @@ def _optional_attr(obj: object, name: str) -> object:
|
|||
return getattr(obj, name, None)
|
||||
|
||||
|
||||
def _delta_has_encrypted_reasoning(delta: object) -> bool:
|
||||
return bool(thinking_blocks_from_reasoning_items(attribute_or_none(delta, "reasoning_items")))
|
||||
|
||||
|
||||
def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]:
|
||||
value: Final = getattr(obj, name, None)
|
||||
return value if value else ()
|
||||
|
|
@ -1518,6 +1523,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
for b in attribute_items(delta, "thinking_blocks")
|
||||
):
|
||||
return False
|
||||
if _delta_has_encrypted_reasoning(delta):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _record_anthropic_stop_fields(self, delta: object) -> None:
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
reasoning_content_from_thinking_blocks,
|
||||
thinking_blocks_with_encrypted_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
||||
parse_tool_call_arguments,
|
||||
|
|
@ -237,6 +238,19 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid
|
|||
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
||||
def _keeps_encrypted_reasoning_for_responses(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.responses.bridge_check import chat_completion_ends_in_responses_api
|
||||
|
||||
return chat_completion_ends_in_responses_api(model, custom_llm_provider)
|
||||
|
||||
|
||||
def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]:
|
||||
return thinking_blocks_with_encrypted_reasoning(
|
||||
getattr(source, "thinking_blocks", None),
|
||||
getattr(source, "reasoning_items", None),
|
||||
)
|
||||
|
||||
|
||||
def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam:
|
||||
eager_input_streaming: Final = eager_input_streaming_flag(tool)
|
||||
if eager_input_streaming is None:
|
||||
|
|
@ -505,7 +519,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
preserve_midturn_system: bool = False,
|
||||
) -> list[AllMessageValues]:
|
||||
new_messages: Final[list[AllMessageValues]] = []
|
||||
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
|
||||
replayable_messages: Final = (
|
||||
messages
|
||||
if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider)
|
||||
else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
|
||||
)
|
||||
leading_count: Final = next(
|
||||
(i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
|
||||
len(replayable_messages),
|
||||
|
|
@ -1443,9 +1461,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for choice in choices
|
||||
):
|
||||
new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks)
|
||||
# Handle thinking blocks first
|
||||
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
for thinking_block in choice.message.thinking_blocks:
|
||||
signed_thinking_blocks = _anthropic_thinking_blocks(choice.message)
|
||||
if signed_thinking_blocks:
|
||||
for thinking_block in signed_thinking_blocks:
|
||||
if thinking_block.get("type") == "thinking":
|
||||
if is_empty_unsigned_thinking_block(thinking_block):
|
||||
continue
|
||||
|
|
@ -1756,24 +1774,24 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
choice.delta
|
||||
) is not None:
|
||||
return "text", TextBlock(type="text", text="")
|
||||
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
|
||||
thinking_blocks = choice.delta.thinking_blocks or []
|
||||
if len(thinking_blocks) > 0:
|
||||
thinking_block = thinking_blocks[0]
|
||||
if thinking_block["type"] == "redacted_thinking":
|
||||
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
|
||||
type="redacted_thinking", data=thinking_block.get("data") or ""
|
||||
)
|
||||
if thinking_block["type"] == "thinking":
|
||||
thinking = thinking_block.get("thinking") or ""
|
||||
signature = thinking_block.get("signature") or ""
|
||||
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)):
|
||||
thinking_block = thinking_blocks[0]
|
||||
if thinking_block["type"] == "redacted_thinking":
|
||||
redacted_data = thinking_block.get("data")
|
||||
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
|
||||
type="redacted_thinking",
|
||||
data=redacted_data if isinstance(redacted_data, str) else "",
|
||||
)
|
||||
if thinking_block["type"] == "thinking":
|
||||
thinking = thinking_block.get("thinking") or ""
|
||||
signature = thinking_block.get("signature") or ""
|
||||
|
||||
assert isinstance(thinking, str)
|
||||
assert isinstance(signature, str)
|
||||
assert isinstance(thinking, str)
|
||||
assert isinstance(signature, str)
|
||||
|
||||
return "thinking", ChatCompletionThinkingBlock(
|
||||
type="thinking", thinking=thinking if signature else "", signature=signature
|
||||
)
|
||||
return "thinking", ChatCompletionThinkingBlock(
|
||||
type="thinking", thinking=thinking if signature else "", signature=signature
|
||||
)
|
||||
# OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
|
||||
# parsers) populate ``reasoning_content`` without ``thinking_blocks``.
|
||||
# ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the
|
||||
|
|
@ -1806,19 +1824,17 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for tool in choice.delta.tool_calls:
|
||||
if tool.function is not None and tool.function.arguments is not None:
|
||||
partial_json = (partial_json or "") + tool.function.arguments
|
||||
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
|
||||
thinking_blocks = choice.delta.thinking_blocks or []
|
||||
if len(thinking_blocks) > 0:
|
||||
for thinking_block in thinking_blocks:
|
||||
if thinking_block["type"] == "thinking":
|
||||
thinking = thinking_block.get("thinking") or ""
|
||||
signature = thinking_block.get("signature") or ""
|
||||
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)):
|
||||
for thinking_block in thinking_blocks:
|
||||
if thinking_block["type"] == "thinking":
|
||||
thinking = thinking_block.get("thinking") or ""
|
||||
signature = thinking_block.get("signature") or ""
|
||||
|
||||
assert isinstance(thinking, str)
|
||||
assert isinstance(signature, str)
|
||||
assert isinstance(thinking, str)
|
||||
assert isinstance(signature, str)
|
||||
|
||||
reasoning_content += thinking
|
||||
reasoning_signature += signature
|
||||
reasoning_content += thinking
|
||||
reasoning_signature += signature
|
||||
# Handle reasoning_content when thinking_blocks is not present
|
||||
# This handles providers like OpenRouter that return reasoning_content
|
||||
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@ from urllib.parse import urlsplit
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_llm_provider,
|
||||
resolve_model_and_provider_for_metadata,
|
||||
)
|
||||
from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
azure_ai_supports_native_responses,
|
||||
|
|
@ -291,3 +294,23 @@ def responses_api_bridge_check(
|
|||
model = model.replace("responses/", "")
|
||||
|
||||
return model_info, model
|
||||
|
||||
|
||||
def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
"""Whether chat completion for this model is served by a Responses API.
|
||||
|
||||
Encrypted reasoning can be replayed only there. A normal chat provider cannot
|
||||
verify the block, so the messages adapter strips it. ``chatgpt`` and a
|
||||
``responses/`` model keep it.
|
||||
"""
|
||||
if model is None and custom_llm_provider is None:
|
||||
return False
|
||||
try:
|
||||
resolved_model, provider = resolve_model_and_provider_for_metadata(model or "", custom_llm_provider)
|
||||
except Exception as error: # noqa: BLE001 # unknown model names must not fail message translation
|
||||
verbose_logger.debug("responses replay check failed for %s: %s", model, error)
|
||||
return custom_llm_provider == LlmProviders.CHATGPT.value or (
|
||||
isinstance(model, str) and (model == "chatgpt" or model.startswith("chatgpt/"))
|
||||
)
|
||||
model_info, _resolved = responses_api_bridge_check(model=resolved_model, custom_llm_provider=provider)
|
||||
return model_info.get("mode") == "responses"
|
||||
|
|
|
|||
|
|
@ -3837,6 +3837,165 @@ def test_reasoning_items_streaming_emitted_on_response_completed():
|
|||
assert ri["summary"][0]["text"] == summary_text
|
||||
|
||||
|
||||
def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keeps():
|
||||
"""A chatgpt /v1/messages stream drops the reasoning item unless this delta carries thinking_blocks.
|
||||
|
||||
response.output_item.done is the only event that has encrypted_content. The chat delta has to
|
||||
expose it as a signed thinking block, and that block alone has to count as stream content.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
encrypted: Final = "gAAAA_round_trip"
|
||||
done: Final = {
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"type": "reasoning",
|
||||
"id": "rs_done",
|
||||
"encrypted_content": encrypted,
|
||||
"summary": [],
|
||||
},
|
||||
}
|
||||
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
).chunk_parser(done)
|
||||
blocks: Final = result.choices[0].delta.thinking_blocks
|
||||
assert blocks is not None
|
||||
assert blocks[0]["type"] == "redacted_thinking"
|
||||
assert blocks[0]["data"] == encrypted_reasoning_signature(encrypted)
|
||||
assert blocks[0]["index"] == 0
|
||||
assert getattr(result.choices[0].delta, "reasoning_content", None) in (None, "")
|
||||
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="chatgpt/gpt-6.1-sol",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="chatgpt",
|
||||
)
|
||||
assert wrapper.is_chunk_non_empty({}, result, {}) is True
|
||||
|
||||
block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block(
|
||||
choices=result.choices
|
||||
)
|
||||
assert block_type == "redacted_thinking"
|
||||
assert start["data"] == encrypted_reasoning_signature(encrypted)
|
||||
|
||||
|
||||
def test_reasoning_output_item_done_with_summary_signs_the_thinking_text():
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
|
||||
encrypted: Final = "gAAAA_with_summary"
|
||||
done: Final = {
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 2,
|
||||
"item": {
|
||||
"type": "reasoning",
|
||||
"id": "rs_sum",
|
||||
"encrypted_content": encrypted,
|
||||
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
|
||||
},
|
||||
}
|
||||
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
).chunk_parser(done)
|
||||
block: Final = result.choices[0].delta.thinking_blocks[0]
|
||||
assert block["type"] == "thinking"
|
||||
assert block["thinking"] == "Oslo is colder."
|
||||
assert block["signature"] == encrypted_reasoning_signature(encrypted)
|
||||
assert block["index"] == 2
|
||||
|
||||
|
||||
def test_reasoning_output_item_done_without_encrypted_content_stays_empty():
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
done: Final = {
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"type": "reasoning",
|
||||
"id": "rs_plain",
|
||||
"summary": [{"type": "summary_text", "text": "already streamed"}],
|
||||
},
|
||||
}
|
||||
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
).chunk_parser(done)
|
||||
assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, [])
|
||||
|
||||
|
||||
def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks():
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
encrypted: Final = "gAAAA_non_stream"
|
||||
reasoning: Final = ResponseReasoningItem(
|
||||
id="rs_tool",
|
||||
summary=[],
|
||||
type="reasoning",
|
||||
content=None,
|
||||
encrypted_content=encrypted,
|
||||
status=None,
|
||||
)
|
||||
function_call: Final = ResponseFunctionToolCall(
|
||||
id="fc_1",
|
||||
type="function_call",
|
||||
status="completed",
|
||||
arguments='{"city":"Oslo"}',
|
||||
call_id="call_1",
|
||||
name="get_weather",
|
||||
)
|
||||
raw: Final = ResponsesAPIResponse.model_construct(
|
||||
id="resp_tool",
|
||||
created_at=1,
|
||||
error=None,
|
||||
incomplete_details=None,
|
||||
model="gpt-6.1-sol",
|
||||
output=[reasoning, function_call],
|
||||
status="completed",
|
||||
usage=None,
|
||||
)
|
||||
model_response: Final = ModelResponse(
|
||||
id="chatcmpl-tool",
|
||||
created=1,
|
||||
model=None,
|
||||
object="chat.completion",
|
||||
system_fingerprint=None,
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
|
||||
)
|
||||
result: Final = LiteLLMResponsesTransformationHandler().transform_response(
|
||||
model="gpt-6.1-sol",
|
||||
raw_response=raw,
|
||||
model_response=model_response,
|
||||
logging_obj=Mock(),
|
||||
request_data={"model": "gpt-6.1-sol"},
|
||||
messages=[{"role": "user", "content": "Which city?"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=Mock(),
|
||||
)
|
||||
message: Final = result.choices[0].message
|
||||
assert message.reasoning_items[0]["encrypted_content"] == encrypted
|
||||
assert message.thinking_blocks[0]["type"] == "redacted_thinking"
|
||||
assert message.thinking_blocks[0]["data"] == encrypted_reasoning_signature(encrypted)
|
||||
assert message.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
|
||||
def test_streaming_function_call_tool_id_for_degenerate_call_id():
|
||||
"""In streaming, Bedrock Mantle's function_call event carries a unique ``id``
|
||||
(``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that
|
||||
|
|
|
|||
|
|
@ -276,6 +276,37 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp
|
|||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_keeps_thinking_blocks_without_other_content(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
"""Encrypted reasoning arrives as thinking_blocks and nothing else, so that delta has to be yielded."""
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chunk-thinking",
|
||||
object="chat.completion.chunk",
|
||||
created=1,
|
||||
model="chatgpt/gpt-6.1-sol",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
finish_reason=None,
|
||||
delta=Delta(
|
||||
thinking_blocks=[
|
||||
{"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"}
|
||||
]
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
assert (
|
||||
initialized_custom_stream_wrapper.is_chunk_non_empty(
|
||||
completion_obj={},
|
||||
model_response=chunk,
|
||||
response_obj={},
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_with_annotations(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -563,6 +563,166 @@ def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning
|
|||
assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"]
|
||||
|
||||
|
||||
def test_chatgpt_messages_keep_encrypted_reasoning_and_replay_it():
|
||||
"""chatgpt /v1/messages ends in the Responses API, so the signed block has to survive into the next request."""
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
|
||||
summary_encrypted: Final = "gAAAA_summary"
|
||||
bytes_encrypted: Final = "gAAAA_bytes"
|
||||
messages: Final = [
|
||||
AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"),
|
||||
AnthopicMessagesAssistantMessageParam(
|
||||
role="assistant",
|
||||
content=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "Oslo is colder.",
|
||||
"signature": encrypted_reasoning_signature(summary_encrypted),
|
||||
},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(bytes_encrypted)},
|
||||
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
|
||||
],
|
||||
),
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}],
|
||||
),
|
||||
]
|
||||
|
||||
kept: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
|
||||
messages=messages,
|
||||
model="chatgpt/gpt-6.1-sol",
|
||||
custom_llm_provider="chatgpt",
|
||||
)
|
||||
assistant: Final = kept[1]
|
||||
assert [block["type"] for block in assistant["thinking_blocks"]] == ["thinking", "redacted_thinking"]
|
||||
assert assistant["thinking_blocks"][0]["signature"] == encrypted_reasoning_signature(summary_encrypted)
|
||||
assert assistant["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted)
|
||||
|
||||
responses_input, _instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
|
||||
kept
|
||||
)
|
||||
reasoning: Final = tuple(item for item in responses_input if item.get("type") == "reasoning")
|
||||
assert [item["encrypted_content"] for item in reasoning] == [summary_encrypted, bytes_encrypted]
|
||||
types: Final = [item.get("type") for item in responses_input]
|
||||
assert types.index("reasoning") < types.index("function_call")
|
||||
|
||||
stripped: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
|
||||
messages=messages,
|
||||
model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert not stripped[1].get("thinking_blocks")
|
||||
|
||||
prefixed: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
|
||||
messages=messages,
|
||||
model="responses/gpt-5-mini",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert prefixed[1]["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted)
|
||||
|
||||
|
||||
def test_translate_openai_reasoning_items_become_signed_thinking_blocks():
|
||||
encrypted: Final = "gAAAA_outbound"
|
||||
openai_choices: Final = [
|
||||
Choices(
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content="Oslo.",
|
||||
reasoning_items=[
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": encrypted,
|
||||
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
|
||||
}
|
||||
],
|
||||
tool_calls=[
|
||||
ChatCompletionAssistantToolCall(
|
||||
id="call_1",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Oslo"}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
|
||||
|
||||
assert result[0]["type"] == "thinking"
|
||||
assert result[0]["thinking"] == "Oslo is colder."
|
||||
assert result[0]["signature"] == encrypted_reasoning_signature(encrypted)
|
||||
assert result[1]["type"] == "text"
|
||||
assert result[2]["type"] == "tool_use"
|
||||
|
||||
|
||||
def test_translate_openai_reasoning_items_without_summary_become_redacted_thinking():
|
||||
encrypted: Final = "gAAAA_bytes_only"
|
||||
openai_choices: Final = [
|
||||
Choices(
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
reasoning_items=[
|
||||
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
|
||||
],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
|
||||
|
||||
assert result == [
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(encrypted)},
|
||||
]
|
||||
|
||||
|
||||
def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_blocks_already_carry_them():
|
||||
encrypted: Final = "gAAAA_once"
|
||||
signature: Final = encrypted_reasoning_signature(encrypted)
|
||||
openai_choices: Final = [
|
||||
Choices(
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content="Oslo.",
|
||||
thinking_blocks=[{"type": "redacted_thinking", "data": signature}],
|
||||
reasoning_items=[
|
||||
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
|
||||
],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
|
||||
|
||||
assert [block["type"] for block in result] == ["redacted_thinking", "text"]
|
||||
|
||||
|
||||
def test_streaming_chunk_with_only_reasoning_items_opens_a_signed_thinking_block():
|
||||
encrypted: Final = "gAAAA_delta"
|
||||
choices: Final = [
|
||||
StreamingChoices(
|
||||
finish_reason=None,
|
||||
index=0,
|
||||
delta=Delta(
|
||||
reasoning_items=[
|
||||
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block(
|
||||
choices=choices
|
||||
)
|
||||
|
||||
assert block_type == "redacted_thinking"
|
||||
assert start["data"] == encrypted_reasoning_signature(encrypted)
|
||||
|
||||
|
||||
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
|
||||
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue