mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(gemini): keep thought signatures for proxy model names without gemini in them (#52212)
* fix(gemini): keep thought signatures for proxy aliases without gemini in the name The proxy ran the non-Gemini signature strip during request setup, before routing, so it judged the alias (g38flash) instead of the deployment (vertex_ai/gemini-3.8-flash) and replaced every replayed signature with Google's placeholder. The strip now runs in the SDK call wrapper, where the model is the deployment's, and skips messages that carry no signature. * fix(gemini): type the tool call id signature strip and match outbound snake_case parts in the test * test(gemini): check signature stripping where the deployment model is known, and that proxy setup keeps it for aliases * test(gemini): cover streamed alias replay and an OpenAI to Vertex alias fallback in one request
This commit is contained in:
parent
98e83feeb7
commit
da296ee4bf
3 changed files with 320 additions and 276 deletions
222
litellm/utils.py
222
litellm/utils.py
|
|
@ -1037,85 +1037,106 @@ def is_gemini_tts_model(model: str) -> bool:
|
|||
return "gemini" in model_name and "tts" in model_name
|
||||
|
||||
|
||||
def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str:
|
||||
"""
|
||||
Remove thought signature from a tool call ID.
|
||||
"""
|
||||
if separator in tool_call_id:
|
||||
return tool_call_id.split(separator, 1)[0]
|
||||
return tool_call_id
|
||||
_THOUGHT_SIGNATURE_CALL_TYPES: Final = frozenset(
|
||||
{CallTypes.completion.value, CallTypes.acompletion.value, CallTypes.anthropic_messages.value}
|
||||
)
|
||||
_MESSAGE_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _process_assistant_message_tool_calls(msg_copy: dict, thought_signature_separator: str) -> dict:
|
||||
"""
|
||||
Process assistant message to remove thought signatures from tool call IDs.
|
||||
"""
|
||||
role: Final = msg_copy.get("role")
|
||||
tool_calls: Final = msg_copy.get("tool_calls")
|
||||
|
||||
if role == "assistant" and isinstance(tool_calls, list):
|
||||
new_tool_calls: Final = []
|
||||
for tc in tool_calls:
|
||||
# Handle both dict and Pydantic model tool calls
|
||||
if hasattr(tc, "model_dump"):
|
||||
# It's a Pydantic model, convert to dict
|
||||
tc_dict = tc.model_dump()
|
||||
elif isinstance(tc, dict):
|
||||
tc_dict = tc.copy()
|
||||
else:
|
||||
new_tool_calls.append(tc)
|
||||
continue
|
||||
|
||||
# Remove thought signature from ID if present
|
||||
if isinstance(tc_dict.get("id"), str):
|
||||
if thought_signature_separator in tc_dict["id"]:
|
||||
tc_dict["id"] = _remove_thought_signature_from_id(tc_dict["id"], thought_signature_separator)
|
||||
|
||||
new_tool_calls.append(tc_dict)
|
||||
msg_copy["tool_calls"] = new_tool_calls
|
||||
|
||||
return msg_copy
|
||||
class _ToolCallIdRef(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
id: str | None = None
|
||||
|
||||
|
||||
def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) -> dict:
|
||||
"""
|
||||
Process tool message to remove thought signature from tool_call_id.
|
||||
"""
|
||||
if msg_copy.get("role") == "tool" and isinstance(msg_copy.get("tool_call_id"), str):
|
||||
if thought_signature_separator in msg_copy["tool_call_id"]:
|
||||
msg_copy["tool_call_id"] = _remove_thought_signature_from_id(
|
||||
msg_copy["tool_call_id"], thought_signature_separator
|
||||
)
|
||||
|
||||
return msg_copy
|
||||
class _MessageToolCallIds(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
tool_call_id: str | None = None
|
||||
tool_calls: tuple[_ToolCallIdRef, ...] | None = None
|
||||
|
||||
|
||||
def _remove_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list:
|
||||
"""
|
||||
Remove thought signatures from tool call IDs in all messages.
|
||||
"""
|
||||
processed_messages: Final = []
|
||||
def _tool_call_ids(message: object) -> tuple[str | None, ...]:
|
||||
try:
|
||||
refs: Final = _MessageToolCallIds.model_validate(message)
|
||||
except ValidationError:
|
||||
return ()
|
||||
return (refs.tool_call_id, *(call.id for call in refs.tool_calls or ()))
|
||||
|
||||
for msg in messages:
|
||||
# Handle Pydantic models (convert to dict)
|
||||
if hasattr(msg, "model_dump"):
|
||||
msg_dict = msg.model_dump()
|
||||
elif isinstance(msg, dict):
|
||||
msg_dict = msg.copy()
|
||||
else:
|
||||
# Unknown type, keep as is
|
||||
processed_messages.append(msg)
|
||||
continue
|
||||
|
||||
# Process assistant messages with tool_calls
|
||||
msg_dict = _process_assistant_message_tool_calls(msg_dict, thought_signature_separator)
|
||||
def _fields(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _MESSAGE_FIELDS.validate_python(value.model_dump() if isinstance(value, BaseModel) else value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
# Process tool messages with tool_call_id
|
||||
msg_dict = _process_tool_message_id(msg_dict, thought_signature_separator)
|
||||
|
||||
processed_messages.append(msg_dict)
|
||||
def _unsigned(value: object, separator: str) -> object:
|
||||
return value.split(separator, 1)[0] if isinstance(value, str) else value
|
||||
|
||||
return processed_messages
|
||||
|
||||
def _unsigned_tool_call(call: object, separator: str) -> object:
|
||||
fields: Final = _fields(call)
|
||||
if fields is None:
|
||||
return call
|
||||
return {key: _unsigned(value, separator) if key == "id" else value for key, value in fields.items()}
|
||||
|
||||
|
||||
def _unsigned_tool_calls(value: object, separator: str) -> object:
|
||||
try:
|
||||
calls: Final = _TOOL_LIST.validate_python(value, strict=True)
|
||||
except ValidationError:
|
||||
return value
|
||||
return [_unsigned_tool_call(call, separator) for call in calls]
|
||||
|
||||
|
||||
def _unsigned_message_field(role: object, key: str, value: object, separator: str) -> object:
|
||||
if role == "assistant" and key == "tool_calls":
|
||||
return _unsigned_tool_calls(value, separator)
|
||||
if role == "tool" and key == "tool_call_id":
|
||||
return _unsigned(value, separator)
|
||||
return value
|
||||
|
||||
|
||||
def _unsigned_message(message: object, separator: str) -> object:
|
||||
fields: Final = _fields(message)
|
||||
if fields is None:
|
||||
return message
|
||||
role: Final = fields.get("role")
|
||||
return {key: _unsigned_message_field(role, key, value, separator) for key, value in fields.items()}
|
||||
|
||||
|
||||
def _target_provider(model: str, declared: object) -> str | None:
|
||||
if isinstance(declared, str) and declared:
|
||||
return declared
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
|
||||
|
||||
try:
|
||||
return resolve_model_and_provider_for_metadata(model=model)[1]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def messages_without_foreign_thought_signatures(
|
||||
call_type: str, model: object, messages: object, custom_llm_provider: object
|
||||
) -> Sequence[object] | None:
|
||||
"""Messages with Gemini thought signatures cut from tool call ids, or None when the call's target keeps them.
|
||||
|
||||
Must run per LLM call with the deployment's model, never on the proxy's alias before routing."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR
|
||||
|
||||
if call_type not in _THOUGHT_SIGNATURE_CALL_TYPES or not isinstance(model, str):
|
||||
return None
|
||||
try:
|
||||
message_list: Final = _TOOL_LIST.validate_python(messages, strict=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
signed: Final = any(
|
||||
tool_call_id is not None and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id
|
||||
for tool_call_id in itertools.chain.from_iterable(map(_tool_call_ids, message_list))
|
||||
)
|
||||
if not signed or _is_gemini_model(model, _target_provider(model, custom_llm_provider)):
|
||||
return None
|
||||
verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model %s", model)
|
||||
return [_unsigned_message(message, THOUGHT_SIGNATURE_SEPARATOR) for message in message_list]
|
||||
|
||||
|
||||
def _restore_correlation_context_if_supported(logging_obj: object) -> None:
|
||||
|
|
@ -1492,55 +1513,6 @@ def function_setup(
|
|||
input=buffer.getvalue(),
|
||||
model=model,
|
||||
)
|
||||
|
||||
### REMOVE THOUGHT SIGNATURES FROM TOOL CALL IDS FOR NON-GEMINI MODELS ###
|
||||
# Gemini models embed thought signatures in tool call IDs. When sending
|
||||
# messages with tool calls to non-Gemini providers, we need to remove these
|
||||
# signatures to ensure compatibility.
|
||||
if isinstance(messages, list) and len(messages) > 0:
|
||||
try:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
resolve_model_and_provider_for_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
)
|
||||
|
||||
# Get custom_llm_provider to determine target provider
|
||||
custom_llm_provider = kwargs.get("custom_llm_provider")
|
||||
|
||||
# If custom_llm_provider not in kwargs, try to determine it from the model
|
||||
metadata_model: Final = _OPAQUE_VALUE.validate_python(model)
|
||||
if not custom_llm_provider and isinstance(metadata_model, str) and metadata_model:
|
||||
try:
|
||||
_, custom_llm_provider = resolve_model_and_provider_for_metadata(
|
||||
model=metadata_model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
except Exception:
|
||||
# If we can't determine the provider, skip this processing
|
||||
pass
|
||||
|
||||
# Only process if target is NOT a Gemini model
|
||||
if not _is_gemini_model(model, custom_llm_provider):
|
||||
verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model")
|
||||
|
||||
# Process messages to remove thought signatures
|
||||
processed_messages: Final = _remove_thought_signatures_from_messages(
|
||||
messages, THOUGHT_SIGNATURE_SEPARATOR
|
||||
)
|
||||
|
||||
# Update messages in kwargs or args
|
||||
if "messages" in kwargs:
|
||||
kwargs["messages"] = processed_messages
|
||||
elif len(args) > 1:
|
||||
args_list: Final = list(args)
|
||||
args_list[1] = processed_messages
|
||||
args = tuple(args_list)
|
||||
|
||||
except Exception as e:
|
||||
# Log the error but don't fail the request
|
||||
verbose_logger.warning("Error removing thought signatures from tool call IDs: %s", e)
|
||||
elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value:
|
||||
messages = args[1] if len(args) > 1 else kwargs.get("input", None)
|
||||
elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value:
|
||||
|
|
@ -2300,6 +2272,14 @@ def client(original_function):
|
|||
|
||||
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
|
||||
assert logging_obj is not None, "logging_obj should not be None after function_setup"
|
||||
signature_free_messages: Final = messages_without_foreign_thought_signatures(
|
||||
call_type,
|
||||
_OPAQUE_VALUE.validate_python(model),
|
||||
_OPAQUE_VALUE.validate_python(kwargs.get("messages")),
|
||||
_OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")),
|
||||
)
|
||||
if signature_free_messages is not None:
|
||||
kwargs["messages"] = signature_free_messages
|
||||
|
||||
## LOAD CREDENTIALS
|
||||
load_credentials_from_list(kwargs, call_type)
|
||||
|
|
@ -2569,6 +2549,14 @@ def client(original_function):
|
|||
|
||||
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
|
||||
assert logging_obj is not None, "logging_obj should not be None after function_setup"
|
||||
signature_free_messages: Final = messages_without_foreign_thought_signatures(
|
||||
call_type,
|
||||
_OPAQUE_VALUE.validate_python(model),
|
||||
_OPAQUE_VALUE.validate_python(kwargs.get("messages")),
|
||||
_OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")),
|
||||
)
|
||||
if signature_free_messages is not None:
|
||||
kwargs["messages"] = signature_free_messages
|
||||
if not _is_litellm_internal_call:
|
||||
bind_budget_reservation_to_callbacks(logging_obj.litellm_params)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
|
|||
from integration._support.client import Gateway, Scenario
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature
|
||||
|
||||
|
|
@ -238,6 +239,152 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat
|
|||
assert response.json()["choices"][0]["message"]["content"] == "done"
|
||||
|
||||
|
||||
_REPLAYED_SIGNATURE: Final = "cmVwbGF5ZWQtc2lnbmF0dXJl"
|
||||
_SIGNED_CALL_ID: Final = f"call_7c1e__thought__{_REPLAYED_SIGNATURE}"
|
||||
_TOOL_TURN_MESSAGES: Final = [
|
||||
{"role": "user", "content": "weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": _SIGNED_CALL_ID,
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": json.dumps(_ARGS)},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": _SIGNED_CALL_ID, "content": "sunny"},
|
||||
]
|
||||
|
||||
|
||||
def _tool_turn_call(gateway: Gateway, model: str, **extra: JsonValue) -> None:
|
||||
response: Final = gateway.client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"tools": [{"type": "function", "function": function} for function in _FUNCTIONS],
|
||||
"messages": _TOOL_TURN_MESSAGES,
|
||||
**extra,
|
||||
},
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
timeout=30,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
|
||||
class _GeminiPart(TypedDict):
|
||||
function_call: NotRequired[ReadOnly[JsonValue]]
|
||||
thoughtSignature: NotRequired[ReadOnly[str]]
|
||||
|
||||
|
||||
class _GeminiContent(TypedDict):
|
||||
parts: ReadOnly[tuple[_GeminiPart, ...]]
|
||||
|
||||
|
||||
class _GeminiRequest(TypedDict):
|
||||
contents: ReadOnly[tuple[_GeminiContent, ...]]
|
||||
|
||||
|
||||
class _ToolCall(TypedDict):
|
||||
id: ReadOnly[str]
|
||||
|
||||
|
||||
class _ChatMessage(TypedDict):
|
||||
tool_calls: NotRequired[ReadOnly[tuple[_ToolCall, ...] | None]]
|
||||
tool_call_id: NotRequired[ReadOnly[str]]
|
||||
|
||||
|
||||
class _ChatRequest(TypedDict):
|
||||
messages: ReadOnly[tuple[_ChatMessage, ...]]
|
||||
|
||||
|
||||
_GEMINI_REQUEST: Final = TypeAdapter(_GeminiRequest)
|
||||
_CHAT_REQUEST: Final = TypeAdapter(_ChatRequest)
|
||||
|
||||
|
||||
def _function_call_signatures(request: Request) -> tuple[str | None, ...]:
|
||||
contents: Final = _GEMINI_REQUEST.validate_json(request.body)["contents"]
|
||||
parts: Final = itertools.chain.from_iterable(content["parts"] for content in contents)
|
||||
return tuple(part.get("thoughtSignature") for part in parts if "function_call" in part)
|
||||
|
||||
|
||||
def _sent_tool_call_ids(request: Request) -> tuple[str, ...]:
|
||||
messages: Final = _CHAT_REQUEST.validate_json(request.body)["messages"]
|
||||
calls: Final = itertools.chain.from_iterable(message.get("tool_calls") or () for message in messages)
|
||||
return (
|
||||
*(call["id"] for call in calls),
|
||||
*(message["tool_call_id"] for message in messages if "tool_call_id" in message),
|
||||
)
|
||||
|
||||
|
||||
def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature(gateway: Gateway) -> None:
|
||||
with (
|
||||
wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = _model(gateway, scenario, wire.url)
|
||||
assert "gemini" not in model
|
||||
_tool_turn_call(gateway, model)
|
||||
(request,) = wire.drain()
|
||||
|
||||
assert request.target == f"{_MODEL_PATH}:generateContent"
|
||||
assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,)
|
||||
|
||||
|
||||
def test_openai_target_receives_tool_call_ids_without_thought_signature(gateway: Gateway) -> None:
|
||||
completion: Final = {
|
||||
"id": "chatcmpl-scripted",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5},
|
||||
}
|
||||
with wire_server(lambda _: Reply(body=json.dumps(completion).encode())) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=f"{wire.url}/v1")
|
||||
_tool_turn_call(gateway, model)
|
||||
(request,) = wire.drain()
|
||||
|
||||
assert request.target == "/v1/chat/completions"
|
||||
assert _sent_tool_call_ids(request) == ("call_7c1e", "call_7c1e")
|
||||
|
||||
|
||||
def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature_streaming(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
payload: Final = json.dumps(_candidate(with_signature=False))
|
||||
with (
|
||||
wire_server(
|
||||
lambda _: Reply(content_type="text/event-stream", chunks=(f"data: {payload}\n\n".encode(),))
|
||||
) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = _model(gateway, scenario, wire.url)
|
||||
_tool_turn_call(gateway, model, stream=True)
|
||||
(request,) = wire.drain()
|
||||
|
||||
assert request.target == f"{_MODEL_PATH}:streamGenerateContent?alt=sse"
|
||||
assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,)
|
||||
|
||||
|
||||
def test_fallback_from_openai_to_vertex_gemini_alias_strips_then_replays_thought_signature(gateway: Gateway) -> None:
|
||||
with (
|
||||
wire_server(lambda _: Reply(status=500, body=b'{"error": {"message": "scripted outage"}}')) as openai_wire,
|
||||
wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as vertex_wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
primary: Final = scenario.model(api_base=f"{openai_wire.url}/v1")
|
||||
fallback: Final = _model(gateway, scenario, vertex_wire.url)
|
||||
_tool_turn_call(gateway, primary, fallbacks=[fallback], num_retries=0)
|
||||
openai_requests: Final = openai_wire.drain()
|
||||
(vertex_request,) = vertex_wire.drain()
|
||||
|
||||
assert openai_requests, "the primary deployment was never tried"
|
||||
assert {_sent_tool_call_ids(request) for request in openai_requests} == {("call_7c1e", "call_7c1e")}
|
||||
assert _function_call_signatures(vertex_request) == (_REPLAYED_SIGNATURE,)
|
||||
|
||||
|
||||
Provider: TypeAlias = Literal["gemini", "vertex_ai"]
|
||||
|
||||
_GEMINI_3_BACKEND: Final = "gemini-3.8-flash"
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelRe
|
|||
from litellm.utils import (
|
||||
Rules,
|
||||
function_setup,
|
||||
messages_without_foreign_thought_signatures,
|
||||
validate_and_fix_openai_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.model_cost_table import publish
|
||||
|
|
@ -321,7 +322,11 @@ def test_ollama_pt_consecutive_user_messages():
|
|||
|
||||
|
||||
def _ollama_tool_turn(*results: object) -> list[dict]:
|
||||
call: Final = {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}}
|
||||
call: Final = {
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
}
|
||||
return [
|
||||
{"role": "user", "content": "Weather in Paris?"},
|
||||
{"role": "assistant", "content": None, "tool_calls": [call]},
|
||||
|
|
@ -1377,7 +1382,6 @@ def test_anthropic_messages_pt_sends_a_signed_block_once_when_thinking_blocks_ho
|
|||
assert result[1] == {"role": "assistant", "content": [signed, redacted, {"type": "text", "text": "hello"}]}
|
||||
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_carry_it():
|
||||
signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"}
|
||||
redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA}
|
||||
|
|
@ -1397,6 +1401,7 @@ def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_ca
|
|||
],
|
||||
}
|
||||
|
||||
|
||||
@REASONING_PLACEMENTS
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_anthropic_messages_pt_drops_cache_control_from_thinking_blocks_only_under_drop_params(
|
||||
|
|
@ -2185,7 +2190,11 @@ def test_convert_to_gemini_tool_call_invoke_drops_unusable_arguments_under_drop_
|
|||
|
||||
function: Final = {"name": "get_weather", "arguments": arguments}
|
||||
message: Final = (
|
||||
{"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function", "function": function}]}
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": function}],
|
||||
}
|
||||
if shape == "tool_calls"
|
||||
else {"role": "assistant", "content": None, "function_call": function}
|
||||
)
|
||||
|
|
@ -2429,10 +2438,11 @@ def test_gemini_tool_media_keeps_vertex_reference_when_inline_data_is_absent(
|
|||
]
|
||||
|
||||
|
||||
|
||||
def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None:
|
||||
message: Final = ChatCompletionToolMessage(
|
||||
role="tool", name="lookup", content=[
|
||||
role="tool",
|
||||
name="lookup",
|
||||
content=[
|
||||
{"type": "text", "text": '{"ok":true}'},
|
||||
{"type": "image_url", "image_url": {"url": "https://8.8.8.8/unavailable.png"}},
|
||||
],
|
||||
|
|
@ -2442,6 +2452,7 @@ def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None:
|
|||
result: Final = convert_to_gemini_tool_call_result(message, None, custom_llm_provider="gemini")
|
||||
assert result == {"function_response": {"name": "lookup", "response": {"ok": True}}}
|
||||
|
||||
|
||||
def test_convert_gemini_tool_call_result_with_anthropic_image_block():
|
||||
"""
|
||||
Test that Anthropic-native image blocks in tool_result list content are
|
||||
|
|
@ -6749,181 +6760,74 @@ def test_empty_content():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_removal_for_non_gemini():
|
||||
"""
|
||||
Test that thought signatures are removed from tool call IDs when sending to non-Gemini models
|
||||
"""
|
||||
rules_obj = Rules()
|
||||
|
||||
# Create messages with thought signatures (as would come from Gemini)
|
||||
messages = [
|
||||
def _signed_tool_turn(*call_ids: str) -> Sequence[Mapping[str, object]]:
|
||||
return [
|
||||
{"role": "user", "content": "What's the weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"location": "SF"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1",
|
||||
"content": "Sunny, 72°F",
|
||||
},
|
||||
]
|
||||
|
||||
# Call function_setup with OpenAI model (non-Gemini)
|
||||
logging_obj, kwargs = function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=rules_obj,
|
||||
start_time=datetime.now(),
|
||||
model="gpt-4",
|
||||
messages=messages,
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
# Verify thought signatures were removed
|
||||
processed_messages = kwargs["messages"]
|
||||
assert processed_messages[1]["tool_calls"][0]["id"] == "call_123"
|
||||
assert processed_messages[2]["tool_call_id"] == "call_123"
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"]
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_preserved_for_gemini():
|
||||
"""
|
||||
Test that thought signatures are preserved when sending to Gemini models
|
||||
"""
|
||||
rules_obj = Rules()
|
||||
|
||||
# Create messages with thought signatures
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"location": "NYC"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2",
|
||||
"content": "Rainy, 65°F",
|
||||
},
|
||||
]
|
||||
|
||||
# Call function_setup with Gemini model
|
||||
logging_obj, kwargs = function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=rules_obj,
|
||||
start_time=datetime.now(),
|
||||
model="gemini-1.5-pro",
|
||||
messages=messages,
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
# Verify thought signatures were preserved (messages should be unchanged)
|
||||
processed_messages = kwargs["messages"]
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"]
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_removal_with_multiple_tool_calls():
|
||||
"""
|
||||
Test that thought signatures are removed from multiple tool calls
|
||||
"""
|
||||
rules_obj = Rules()
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Get weather and time"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1",
|
||||
"id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2",
|
||||
"type": "function",
|
||||
"function": {"name": "get_time", "arguments": "{}"},
|
||||
},
|
||||
}
|
||||
for index, call_id in enumerate(call_ids)
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1",
|
||||
"content": "Sunny",
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2",
|
||||
"content": "3:00 PM",
|
||||
},
|
||||
*(
|
||||
{"role": "tool", "tool_call_id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}", "content": "Sunny"}
|
||||
for index, call_id in enumerate(call_ids)
|
||||
),
|
||||
]
|
||||
|
||||
logging_obj, kwargs = function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=rules_obj,
|
||||
start_time=datetime.now(),
|
||||
model="claude-3-opus",
|
||||
messages=messages,
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
custom_llm_provider="anthropic",
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("gpt-4", "openai"), ("claude-3-opus", "anthropic"), ("openai/gpt-4", None)],
|
||||
)
|
||||
def test_thought_signatures_are_cut_from_every_tool_call_id_for_a_non_gemini_target(model, custom_llm_provider):
|
||||
stripped = messages_without_foreign_thought_signatures(
|
||||
"acompletion", model, _signed_tool_turn("call_1", "call_2"), custom_llm_provider
|
||||
)
|
||||
|
||||
processed_messages = kwargs["messages"]
|
||||
assert stripped is not None
|
||||
assert [call["id"] for call in stripped[1]["tool_calls"]] == ["call_1", "call_2"]
|
||||
assert [message["tool_call_id"] for message in stripped[2:]] == ["call_1", "call_2"]
|
||||
|
||||
# Check all tool call IDs are cleaned
|
||||
assert processed_messages[1]["tool_calls"][0]["id"] == "call_1"
|
||||
assert processed_messages[1]["tool_calls"][1]["id"] == "call_2"
|
||||
assert processed_messages[2]["tool_call_id"] == "call_1"
|
||||
assert processed_messages[3]["tool_call_id"] == "call_2"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("gemini-1.5-pro", "vertex_ai"), ("vertex_ai/gemini-3.8-flash", None), ("gemini/gemini-3.8-flash", None)],
|
||||
)
|
||||
def test_thought_signatures_are_kept_for_a_gemini_target(model, custom_llm_provider):
|
||||
assert (
|
||||
messages_without_foreign_thought_signatures(
|
||||
"acompletion", model, _signed_tool_turn("call_1"), custom_llm_provider
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_messages_without_signed_tool_call_ids_are_left_alone_for_a_non_gemini_target():
|
||||
messages = [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!"}]
|
||||
|
||||
assert messages_without_foreign_thought_signatures("acompletion", "gpt-4", messages, "openai") is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_messages_without_tool_calls_unchanged():
|
||||
"""
|
||||
Test that messages without tool calls pass through unchanged
|
||||
"""
|
||||
rules_obj = Rules()
|
||||
def test_proxy_request_setup_keeps_signatures_for_a_model_name_without_gemini_in_it():
|
||||
messages = _signed_tool_turn("call_1")
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
|
||||
logging_obj, kwargs = function_setup(
|
||||
_, kwargs = function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=rules_obj,
|
||||
rules_obj=Rules(),
|
||||
start_time=datetime.now(),
|
||||
model="gpt-4",
|
||||
model="g38flash",
|
||||
messages=messages,
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
# Messages should be unchanged
|
||||
assert kwargs["messages"] == messages
|
||||
assert kwargs["messages"] is messages, "the proxy alias must not decide signature stripping before routing"
|
||||
|
||||
|
||||
def test_llama_3_prompt():
|
||||
|
|
@ -8990,7 +8894,6 @@ def test_anthropic_messages_pt_interleave_more_thinking_than_tool_groups():
|
|||
], f"Expected order but got: {types}"
|
||||
|
||||
|
||||
|
||||
def _paused_web_search_turn(
|
||||
thinking_blocks: list[dict[str, str]] | None, content: str | list[dict[str, str]] = "Step 1 is Tokyo."
|
||||
) -> dict[str, object]:
|
||||
|
|
@ -9453,7 +9356,11 @@ TEMPLATE_INITIAL = "<start>"
|
|||
TEMPLATE_FINAL = "<end>"
|
||||
|
||||
PARTIAL_TEMPLATE = {"roles": TEMPLATE_ROLES}
|
||||
FULL_TEMPLATE = {"roles": TEMPLATE_ROLES, "initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL}
|
||||
FULL_TEMPLATE = {
|
||||
"roles": TEMPLATE_ROLES,
|
||||
"initial_prompt_value": TEMPLATE_INITIAL,
|
||||
"final_prompt_value": TEMPLATE_FINAL,
|
||||
}
|
||||
ROLELESS_TEMPLATE = {"initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL}
|
||||
|
||||
|
||||
|
|
@ -9675,7 +9582,9 @@ def test_template_wraps_only_the_roles_it_defines_across_a_conversation(monkeypa
|
|||
pytest.param({"roles": {"user": {**TEMPLATE_ROLES["user"], "priority": 1}}}, id="extra-role-key"),
|
||||
pytest.param({"roles": {**TEMPLATE_ROLES, "tool": {"pre_message": 5}}}, id="unused-role-wrong-type"),
|
||||
pytest.param({"roles": {**TEMPLATE_ROLES, 1: {"pre_message": "x"}}}, id="unused-non-string-role"),
|
||||
pytest.param({"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys"),
|
||||
pytest.param(
|
||||
{"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_template_shapes_main_rendered_keep_rendering(template, monkeypatch):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue