mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 4b66255950 into ff3983c1f6
This commit is contained in:
commit
d83d3e1b98
5 changed files with 169 additions and 12 deletions
|
|
@ -720,8 +720,54 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
if chat_reasoning is not None and chat_reasoning.summary is not None:
|
||||
completion_kwargs["reasoning_summary"] = chat_reasoning.summary
|
||||
|
||||
if LiteLLMMessagesToCompletionTransformationHandler._uses_responses_bridge(completion_kwargs):
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.types.llms.anthropic import AllAnthropicPassThroughMessageValues, AnthropicMessagesRequest
|
||||
|
||||
adapter: Final = LiteLLMAnthropicMessagesAdapter()
|
||||
replay_messages: Final = adapter.translate_anthropic_messages_to_openai(
|
||||
cast(list[AllAnthropicPassThroughMessageValues], messages),
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
preserve_encrypted_reasoning=True,
|
||||
)
|
||||
replay_request: Final = AnthropicMessagesRequest(model=model, messages=messages, system=system)
|
||||
adapter._add_system_message_to_messages(replay_messages, replay_request)
|
||||
completion_kwargs["messages"] = replay_messages
|
||||
|
||||
return completion_kwargs, tool_name_mapping
|
||||
|
||||
@staticmethod
|
||||
def _uses_responses_bridge(completion_kwargs: _CompletionKwargs) -> bool:
|
||||
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
|
||||
|
||||
model: Final = completion_kwargs.get("model")
|
||||
if not isinstance(model, str):
|
||||
return False
|
||||
provider: Final = completion_kwargs.get("custom_llm_provider")
|
||||
api_base: Final = completion_kwargs.get("api_base")
|
||||
try:
|
||||
local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=provider if isinstance(provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
)
|
||||
except (ValueError, litellm.BadRequestError):
|
||||
return False
|
||||
tools: Final = completion_kwargs.get("tools")
|
||||
web_search: Final = completion_kwargs.get("web_search_options")
|
||||
effort: Final = completion_kwargs.get("reasoning_effort")
|
||||
model_info, _ = responses_api_bridge_check(
|
||||
model=local_model,
|
||||
custom_llm_provider=resolved_provider,
|
||||
tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None,
|
||||
web_search_options=cast(OpenAIWebSearchOptions, web_search) if isinstance(web_search, dict) else None,
|
||||
reasoning_effort=effort if isinstance(effort, str) else str_keyed_mapping(effort),
|
||||
api_base=resolved_api_base,
|
||||
deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")),
|
||||
)
|
||||
return model_info.get("mode") == "responses"
|
||||
|
||||
@staticmethod
|
||||
async def async_anthropic_messages_handler(
|
||||
max_tokens: int,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm._uuid import uuid
|
|||
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_params
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -494,9 +495,10 @@ class _CombinedChunkSplitter:
|
|||
)
|
||||
|
||||
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
|
||||
normalized: Final = self._with_reasoning_items(chunk)
|
||||
split_chunks: Final = tuple(
|
||||
split_chunk
|
||||
for combined_chunk in self._split(chunk)
|
||||
for combined_chunk in self._split(normalized)
|
||||
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
|
||||
)
|
||||
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
|
||||
|
|
@ -540,6 +542,26 @@ class _CombinedChunkSplitter:
|
|||
def _restates_unsigned_thinking(self, text: str) -> bool:
|
||||
return len(text) == self._unsigned_thinking_length and text == self._unsigned_thinking.getvalue()
|
||||
|
||||
@staticmethod
|
||||
def _with_reasoning_items(chunk: "ModelResponseStream") -> "ModelResponseStream":
|
||||
if not chunk.choices:
|
||||
return chunk
|
||||
choice: Final = chunk.choices[0]
|
||||
blocks: Final = tuple(
|
||||
block
|
||||
for item in (choice.delta.reasoning_items or ())
|
||||
if (
|
||||
block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item(
|
||||
(), item.get("encrypted_content")
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
if not blocks:
|
||||
return chunk
|
||||
delta: Final = choice.delta.model_copy(update={"thinking_blocks": list(blocks)})
|
||||
return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta})]})
|
||||
|
||||
def __iter__(self) -> "Iterator[ModelResponseStream]":
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ from litellm.llms.anthropic.messages.native.utils import (
|
|||
refusal_reply_text,
|
||||
refusal_stop_details,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
AllAnthropicPassThroughMessageValues,
|
||||
|
|
@ -495,9 +496,14 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
preserve_midturn_system: bool = False,
|
||||
preserve_encrypted_reasoning: bool = False,
|
||||
) -> list[AllMessageValues]:
|
||||
new_messages: Final[list[AllMessageValues]] = []
|
||||
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
|
||||
replayable_messages: Final = (
|
||||
messages
|
||||
if preserve_encrypted_reasoning
|
||||
else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
|
||||
)
|
||||
leading_count: Final = next(
|
||||
(i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
|
||||
len(replayable_messages),
|
||||
|
|
@ -1435,8 +1441,19 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for choice in choices
|
||||
):
|
||||
new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks)
|
||||
# Handle thinking blocks first
|
||||
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
reasoning_blocks: Final = tuple(
|
||||
block
|
||||
for item in (choice.message.reasoning_items or ())
|
||||
if (
|
||||
block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item(
|
||||
item.get("summary", ()), item.get("encrypted_content")
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
if reasoning_blocks:
|
||||
new_content.extend(reasoning_blocks)
|
||||
elif hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
for thinking_block in choice.message.thinking_blocks:
|
||||
if thinking_block.get("type") == "thinking":
|
||||
if is_empty_unsigned_thinking_block(thinking_block):
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ import pytest
|
|||
import respx
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
encrypted_reasoning_signature,
|
||||
|
|
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
_bedrock_converse_messages_pt,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
AnthropicAdapter,
|
||||
|
|
@ -37,7 +38,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthopicMessagesAssistantMessageParam,
|
||||
AnthropicMessagesUserMessageParam,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionAssistantToolCall
|
||||
from litellm.types.llms.openai import ChatCompletionAssistantToolCall, ChatCompletionReasoningSummaryTextBlock
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
Choices,
|
||||
|
|
@ -52,6 +53,52 @@ from litellm.types.utils import (
|
|||
from litellm.utils import remove_additional_properties, remove_strict_from_schema
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["chatgpt/gpt-5", "openai/responses/gpt-5"])
|
||||
@pytest.mark.parametrize("summary", [[], [{"type": "summary_text", "text": "Check the weather"}]])
|
||||
def test_responses_reasoning_round_trips_through_anthropic_tool_turn(
|
||||
model: str, summary: list[ChatCompletionReasoningSummaryTextBlock]
|
||||
) -> None:
|
||||
response: Final = ModelResponse(
|
||||
model=model,
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"reasoning_items": [{"type": "reasoning", "summary": summary, "encrypted_content": "opaque-test"}],
|
||||
"tool_calls": [
|
||||
{"id": "call_weather", "type": "function", "function": {"name": "weather", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
|
||||
assert result["content"][0] == (
|
||||
{"type": "thinking", "thinking": "Check the weather", "signature": encrypted_reasoning_signature("opaque-test")}
|
||||
if summary
|
||||
else {"type": "redacted_thinking", "data": encrypted_reasoning_signature("opaque-test")}
|
||||
)
|
||||
prepared, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
||||
model=model,
|
||||
max_tokens=100,
|
||||
system="Keep the system instruction",
|
||||
messages=[
|
||||
{"role": "user", "content": "Weather?"},
|
||||
{"role": "assistant", "content": result["content"]},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_weather", "content": "sunny"}]},
|
||||
],
|
||||
)
|
||||
items, instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
|
||||
prepared["messages"]
|
||||
)
|
||||
assert instructions == "Keep the system instruction"
|
||||
assert [item["type"] for item in items] == ["message", "reasoning", "function_call", "function_call_output"]
|
||||
assert items[1]["encrypted_content"] == "opaque-test"
|
||||
|
||||
|
||||
def test_translate_openai_response_to_anthropic_empty_choices() -> None:
|
||||
response: Final = ModelResponse(
|
||||
id="chatcmpl-empty",
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
convert_to_streaming_response,
|
||||
convert_to_streaming_response_async,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
from litellm.llms.anthropic.messages.adapters.streaming_iterator import (
|
||||
AnthropicStreamWrapper,
|
||||
_CombinedChunkSplitter,
|
||||
|
|
@ -39,6 +40,27 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
|
||||
async def _stream_with_encrypted_reasoning() -> AsyncIterator[ModelResponseStream]:
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content="Visible summary"))])
|
||||
yield ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(reasoning_items=[{"type": "reasoning", "summary": [], "encrypted_content": "opaque-test"}]),
|
||||
finish_reason="stop",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_stream_replays_encrypted_reasoning_without_repeating_summary() -> None:
|
||||
content: Final = _anthropic_sdk_final_content(_stream_with_encrypted_reasoning())
|
||||
assert content == [
|
||||
{"type": "thinking", "thinking": "Visible summary", "signature": ""},
|
||||
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("opaque-test")},
|
||||
]
|
||||
|
||||
|
||||
def _build_fake_stream(content: str, finish_reason: str = "stop") -> MockResponseIterator:
|
||||
"""Mimic a Vertex Gemma `:predict` fake stream: one collapsed chunk."""
|
||||
model_response = ModelResponse()
|
||||
|
|
@ -346,9 +368,7 @@ def _anthropic_sdk_final_content(
|
|||
lambda request: httpx.Response(200, content=sse.encode(), headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
client: Final = anthropic.Anthropic(api_key="transport-only", http_client=httpx.Client(transport=transport))
|
||||
with client.messages.stream(
|
||||
model="claude", max_tokens=16, messages=[{"role": "user", "content": "hi"}]
|
||||
) as stream:
|
||||
with client.messages.stream(model="claude", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) as stream:
|
||||
return [block.model_dump(exclude_none=True) for block in stream.get_final_message().content]
|
||||
|
||||
|
||||
|
|
@ -377,7 +397,12 @@ def test_fake_stream_gives_each_thinking_block_its_own_content_block(thinking_bl
|
|||
|
||||
|
||||
def _native_thinking_chunk(thinking: str, index: int, signature: str | None = None) -> ModelResponseStream:
|
||||
block: Final = {"type": "thinking", "thinking": thinking, "index": index, **({"signature": signature} if signature else {})}
|
||||
block: Final = {
|
||||
"type": "thinking",
|
||||
"thinking": thinking,
|
||||
"index": index,
|
||||
**({"signature": signature} if signature else {}),
|
||||
}
|
||||
return ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=thinking or None, thinking_blocks=[block]))]
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue