fix(gemini): keep thought signatures for proxy model names without gemini in them (#52212)

* fix(gemini): keep thought signatures for proxy aliases without gemini in the name

The proxy ran the non-Gemini signature strip during request setup, before
routing, so it judged the alias (g38flash) instead of the deployment
(vertex_ai/gemini-3.8-flash) and replaced every replayed signature with
Google's placeholder. The strip now runs in the SDK call wrapper, where the
model is the deployment's, and skips messages that carry no signature.

* fix(gemini): type the tool call id signature strip and match outbound snake_case parts in the test

* test(gemini): check signature stripping where the deployment model is known, and that proxy setup keeps it for aliases

* test(gemini): cover streamed alias replay and an OpenAI to Vertex alias fallback in one request
This commit is contained in:
ryan-crabbe-berri 2026-11-16 22:20:51 -08:00 • committed by GitHub
parent 98e83feeb7
commit da296ee4bf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 320 additions and 276 deletions

View file

@ -1037,85 +1037,106 @@ def is_gemini_tts_model(model: str) -> bool:
return "gemini" in model_name and "tts" in model_name
def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str:
"""
Remove thought signature from a tool call ID.
"""
if separator in tool_call_id:
return tool_call_id.split(separator, 1)[0]
return tool_call_id
_THOUGHT_SIGNATURE_CALL_TYPES: Final = frozenset(
{CallTypes.completion.value, CallTypes.acompletion.value, CallTypes.anthropic_messages.value}
)
_MESSAGE_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _process_assistant_message_tool_calls(msg_copy: dict, thought_signature_separator: str) -> dict:
"""
Process assistant message to remove thought signatures from tool call IDs.
"""
role: Final = msg_copy.get("role")
tool_calls: Final = msg_copy.get("tool_calls")
if role == "assistant" and isinstance(tool_calls, list):
new_tool_calls: Final = []
for tc in tool_calls:
# Handle both dict and Pydantic model tool calls
if hasattr(tc, "model_dump"):
# It's a Pydantic model, convert to dict
tc_dict = tc.model_dump()
elif isinstance(tc, dict):
tc_dict = tc.copy()
else:
new_tool_calls.append(tc)
continue
# Remove thought signature from ID if present
if isinstance(tc_dict.get("id"), str):
if thought_signature_separator in tc_dict["id"]:
tc_dict["id"] = _remove_thought_signature_from_id(tc_dict["id"], thought_signature_separator)
new_tool_calls.append(tc_dict)
msg_copy["tool_calls"] = new_tool_calls
return msg_copy
class _ToolCallIdRef(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
id: str | None = None
def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) -> dict:
"""
Process tool message to remove thought signature from tool_call_id.
"""
if msg_copy.get("role") == "tool" and isinstance(msg_copy.get("tool_call_id"), str):
if thought_signature_separator in msg_copy["tool_call_id"]:
msg_copy["tool_call_id"] = _remove_thought_signature_from_id(
msg_copy["tool_call_id"], thought_signature_separator
)
return msg_copy
class _MessageToolCallIds(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
tool_call_id: str | None = None
tool_calls: tuple[_ToolCallIdRef, ...] | None = None
def _remove_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list:
"""
Remove thought signatures from tool call IDs in all messages.
"""
processed_messages: Final = []
def _tool_call_ids(message: object) -> tuple[str | None, ...]:
try:
refs: Final = _MessageToolCallIds.model_validate(message)
except ValidationError:
return ()
return (refs.tool_call_id, *(call.id for call in refs.tool_calls or ()))
for msg in messages:
# Handle Pydantic models (convert to dict)
if hasattr(msg, "model_dump"):
msg_dict = msg.model_dump()
elif isinstance(msg, dict):
msg_dict = msg.copy()
else:
# Unknown type, keep as is
processed_messages.append(msg)
continue
# Process assistant messages with tool_calls
msg_dict = _process_assistant_message_tool_calls(msg_dict, thought_signature_separator)
def _fields(value: object) -> Mapping[str, object] | None:
try:
return _MESSAGE_FIELDS.validate_python(value.model_dump() if isinstance(value, BaseModel) else value)
except ValidationError:
return None
# Process tool messages with tool_call_id
msg_dict = _process_tool_message_id(msg_dict, thought_signature_separator)
processed_messages.append(msg_dict)
def _unsigned(value: object, separator: str) -> object:
return value.split(separator, 1)[0] if isinstance(value, str) else value
return processed_messages
def _unsigned_tool_call(call: object, separator: str) -> object:
fields: Final = _fields(call)
if fields is None:
return call
return {key: _unsigned(value, separator) if key == "id" else value for key, value in fields.items()}
def _unsigned_tool_calls(value: object, separator: str) -> object:
try:
calls: Final = _TOOL_LIST.validate_python(value, strict=True)
except ValidationError:
return value
return [_unsigned_tool_call(call, separator) for call in calls]
def _unsigned_message_field(role: object, key: str, value: object, separator: str) -> object:
if role == "assistant" and key == "tool_calls":
return _unsigned_tool_calls(value, separator)
if role == "tool" and key == "tool_call_id":
return _unsigned(value, separator)
return value
def _unsigned_message(message: object, separator: str) -> object:
fields: Final = _fields(message)
if fields is None:
return message
role: Final = fields.get("role")
return {key: _unsigned_message_field(role, key, value, separator) for key, value in fields.items()}
def _target_provider(model: str, declared: object) -> str | None:
if isinstance(declared, str) and declared:
return declared
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
try:
return resolve_model_and_provider_for_metadata(model=model)[1]
except Exception:
return None
def messages_without_foreign_thought_signatures(
call_type: str, model: object, messages: object, custom_llm_provider: object
) -> Sequence[object] | None:
"""Messages with Gemini thought signatures cut from tool call ids, or None when the call's target keeps them.
Must run per LLM call with the deployment's model, never on the proxy's alias before routing."""
from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR
if call_type not in _THOUGHT_SIGNATURE_CALL_TYPES or not isinstance(model, str):
return None
try:
message_list: Final = _TOOL_LIST.validate_python(messages, strict=True)
except ValidationError:
return None
signed: Final = any(
tool_call_id is not None and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id
for tool_call_id in itertools.chain.from_iterable(map(_tool_call_ids, message_list))
)
if not signed or _is_gemini_model(model, _target_provider(model, custom_llm_provider)):
return None
verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model %s", model)
return [_unsigned_message(message, THOUGHT_SIGNATURE_SEPARATOR) for message in message_list]
def _restore_correlation_context_if_supported(logging_obj: object) -> None:
@ -1492,55 +1513,6 @@ def function_setup(
input=buffer.getvalue(),
model=model,
)
### REMOVE THOUGHT SIGNATURES FROM TOOL CALL IDS FOR NON-GEMINI MODELS ###
# Gemini models embed thought signatures in tool call IDs. When sending
# messages with tool calls to non-Gemini providers, we need to remove these
# signatures to ensure compatibility.
if isinstance(messages, list) and len(messages) > 0:
try:
from litellm.litellm_core_utils.get_llm_provider_logic import (
resolve_model_and_provider_for_metadata,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
)
# Get custom_llm_provider to determine target provider
custom_llm_provider = kwargs.get("custom_llm_provider")
# If custom_llm_provider not in kwargs, try to determine it from the model
metadata_model: Final = _OPAQUE_VALUE.validate_python(model)
if not custom_llm_provider and isinstance(metadata_model, str) and metadata_model:
try:
_, custom_llm_provider = resolve_model_and_provider_for_metadata(
model=metadata_model,
custom_llm_provider=custom_llm_provider,
)
except Exception:
# If we can't determine the provider, skip this processing
pass
# Only process if target is NOT a Gemini model
if not _is_gemini_model(model, custom_llm_provider):
verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model")
# Process messages to remove thought signatures
processed_messages: Final = _remove_thought_signatures_from_messages(
messages, THOUGHT_SIGNATURE_SEPARATOR
)
# Update messages in kwargs or args
if "messages" in kwargs:
kwargs["messages"] = processed_messages
elif len(args) > 1:
args_list: Final = list(args)
args_list[1] = processed_messages
args = tuple(args_list)
except Exception as e:
# Log the error but don't fail the request
verbose_logger.warning("Error removing thought signatures from tool call IDs: %s", e)
elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value:
messages = args[1] if len(args) > 1 else kwargs.get("input", None)
elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value:
@ -2300,6 +2272,14 @@ def client(original_function):
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
assert logging_obj is not None, "logging_obj should not be None after function_setup"
signature_free_messages: Final = messages_without_foreign_thought_signatures(
call_type,
_OPAQUE_VALUE.validate_python(model),
_OPAQUE_VALUE.validate_python(kwargs.get("messages")),
_OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")),
)
if signature_free_messages is not None:
kwargs["messages"] = signature_free_messages
## LOAD CREDENTIALS
load_credentials_from_list(kwargs, call_type)
@ -2569,6 +2549,14 @@ def client(original_function):
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
assert logging_obj is not None, "logging_obj should not be None after function_setup"
signature_free_messages: Final = messages_without_foreign_thought_signatures(
call_type,
_OPAQUE_VALUE.validate_python(model),
_OPAQUE_VALUE.validate_python(kwargs.get("messages")),
_OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")),
)
if signature_free_messages is not None:
kwargs["messages"] = signature_free_messages
if not _is_litellm_internal_call:
bind_budget_reservation_to_callbacks(logging_obj.litellm_params)

View file

@ -13,6 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from integration._support.client import Gateway, Scenario
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature
@ -238,6 +239,152 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat
assert response.json()["choices"][0]["message"]["content"] == "done"
_REPLAYED_SIGNATURE: Final = "cmVwbGF5ZWQtc2lnbmF0dXJl"
_SIGNED_CALL_ID: Final = f"call_7c1e__thought__{_REPLAYED_SIGNATURE}"
_TOOL_TURN_MESSAGES: Final = [
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": _SIGNED_CALL_ID,
"type": "function",
"function": {"name": "get_weather", "arguments": json.dumps(_ARGS)},
}
],
},
{"role": "tool", "tool_call_id": _SIGNED_CALL_ID, "content": "sunny"},
]
def _tool_turn_call(gateway: Gateway, model: str, **extra: JsonValue) -> None:
response: Final = gateway.client.post(
"/v1/chat/completions",
json={
"model": model,
"tools": [{"type": "function", "function": function} for function in _FUNCTIONS],
"messages": _TOOL_TURN_MESSAGES,
**extra,
},
headers={"Authorization": f"Bearer {gateway.key}"},
timeout=30,
)
assert response.status_code == 200, response.text
class _GeminiPart(TypedDict):
function_call: NotRequired[ReadOnly[JsonValue]]
thoughtSignature: NotRequired[ReadOnly[str]]
class _GeminiContent(TypedDict):
parts: ReadOnly[tuple[_GeminiPart, ...]]
class _GeminiRequest(TypedDict):
contents: ReadOnly[tuple[_GeminiContent, ...]]
class _ToolCall(TypedDict):
id: ReadOnly[str]
class _ChatMessage(TypedDict):
tool_calls: NotRequired[ReadOnly[tuple[_ToolCall, ...] | None]]
tool_call_id: NotRequired[ReadOnly[str]]
class _ChatRequest(TypedDict):
messages: ReadOnly[tuple[_ChatMessage, ...]]
_GEMINI_REQUEST: Final = TypeAdapter(_GeminiRequest)
_CHAT_REQUEST: Final = TypeAdapter(_ChatRequest)
def _function_call_signatures(request: Request) -> tuple[str | None, ...]:
contents: Final = _GEMINI_REQUEST.validate_json(request.body)["contents"]
parts: Final = itertools.chain.from_iterable(content["parts"] for content in contents)
return tuple(part.get("thoughtSignature") for part in parts if "function_call" in part)
def _sent_tool_call_ids(request: Request) -> tuple[str, ...]:
messages: Final = _CHAT_REQUEST.validate_json(request.body)["messages"]
calls: Final = itertools.chain.from_iterable(message.get("tool_calls") or () for message in messages)
return (
*(call["id"] for call in calls),
*(message["tool_call_id"] for message in messages if "tool_call_id" in message),
)
def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature(gateway: Gateway) -> None:
with (
wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as wire,
gateway.scenario() as scenario,
):
model: Final = _model(gateway, scenario, wire.url)
assert "gemini" not in model
_tool_turn_call(gateway, model)
(request,) = wire.drain()
assert request.target == f"{_MODEL_PATH}:generateContent"
assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,)
def test_openai_target_receives_tool_call_ids_without_thought_signature(gateway: Gateway) -> None:
completion: Final = {
"id": "chatcmpl-scripted",
"object": "chat.completion",
"created": 0,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5},
}
with wire_server(lambda _: Reply(body=json.dumps(completion).encode())) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(api_base=f"{wire.url}/v1")
_tool_turn_call(gateway, model)
(request,) = wire.drain()
assert request.target == "/v1/chat/completions"
assert _sent_tool_call_ids(request) == ("call_7c1e", "call_7c1e")
def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature_streaming(
gateway: Gateway,
) -> None:
payload: Final = json.dumps(_candidate(with_signature=False))
with (
wire_server(
lambda _: Reply(content_type="text/event-stream", chunks=(f"data: {payload}\n\n".encode(),))
) as wire,
gateway.scenario() as scenario,
):
model: Final = _model(gateway, scenario, wire.url)
_tool_turn_call(gateway, model, stream=True)
(request,) = wire.drain()
assert request.target == f"{_MODEL_PATH}:streamGenerateContent?alt=sse"
assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,)
def test_fallback_from_openai_to_vertex_gemini_alias_strips_then_replays_thought_signature(gateway: Gateway) -> None:
with (
wire_server(lambda _: Reply(status=500, body=b'{"error": {"message": "scripted outage"}}')) as openai_wire,
wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as vertex_wire,
gateway.scenario() as scenario,
):
primary: Final = scenario.model(api_base=f"{openai_wire.url}/v1")
fallback: Final = _model(gateway, scenario, vertex_wire.url)
_tool_turn_call(gateway, primary, fallbacks=[fallback], num_retries=0)
openai_requests: Final = openai_wire.drain()
(vertex_request,) = vertex_wire.drain()
assert openai_requests, "the primary deployment was never tried"
assert {_sent_tool_call_ids(request) for request in openai_requests} == {("call_7c1e", "call_7c1e")}
assert _function_call_signatures(vertex_request) == (_REPLAYED_SIGNATURE,)
Provider: TypeAlias = Literal["gemini", "vertex_ai"]
_GEMINI_3_BACKEND: Final = "gemini-3.8-flash"

View file

@ -85,6 +85,7 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelRe
from litellm.utils import (
Rules,
function_setup,
messages_without_foreign_thought_signatures,
validate_and_fix_openai_messages,
)
from litellm.litellm_core_utils.model_cost_table import publish
@ -321,7 +322,11 @@ def test_ollama_pt_consecutive_user_messages():
def _ollama_tool_turn(*results: object) -> list[dict]:
call: Final = {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}}
call: Final = {
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
return [
{"role": "user", "content": "Weather in Paris?"},
{"role": "assistant", "content": None, "tool_calls": [call]},
@ -1377,7 +1382,6 @@ def test_anthropic_messages_pt_sends_a_signed_block_once_when_thinking_blocks_ho
assert result[1] == {"role": "assistant", "content": [signed, redacted, {"type": "text", "text": "hello"}]}
def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_carry_it():
signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"}
redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA}
@ -1397,6 +1401,7 @@ def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_ca
],
}
@REASONING_PLACEMENTS
@pytest.mark.parametrize("drop_params", [False, True])
def test_anthropic_messages_pt_drops_cache_control_from_thinking_blocks_only_under_drop_params(
@ -2185,7 +2190,11 @@ def test_convert_to_gemini_tool_call_invoke_drops_unusable_arguments_under_drop_
function: Final = {"name": "get_weather", "arguments": arguments}
message: Final = (
{"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function", "function": function}]}
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": function}],
}
if shape == "tool_calls"
else {"role": "assistant", "content": None, "function_call": function}
)
@ -2429,10 +2438,11 @@ def test_gemini_tool_media_keeps_vertex_reference_when_inline_data_is_absent(
]
def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None:
message: Final = ChatCompletionToolMessage(
role="tool", name="lookup", content=[
role="tool",
name="lookup",
content=[
{"type": "text", "text": '{"ok":true}'},
{"type": "image_url", "image_url": {"url": "https://8.8.8.8/unavailable.png"}},
],
@ -2442,6 +2452,7 @@ def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None:
result: Final = convert_to_gemini_tool_call_result(message, None, custom_llm_provider="gemini")
assert result == {"function_response": {"name": "lookup", "response": {"ok": True}}}
def test_convert_gemini_tool_call_result_with_anthropic_image_block():
"""
Test that Anthropic-native image blocks in tool_result list content are
@ -6749,181 +6760,74 @@ def test_empty_content():
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_thought_signature_removal_for_non_gemini():
"""
Test that thought signatures are removed from tool call IDs when sending to non-Gemini models
"""
rules_obj = Rules()
# Create messages with thought signatures (as would come from Gemini)
messages = [
def _signed_tool_turn(*call_ids: str) -> Sequence[Mapping[str, object]]:
return [
{"role": "user", "content": "What's the weather?"},
{
"role": "assistant",
"tool_calls": [
{
"id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"location": "SF"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1",
"content": "Sunny, 72°F",
},
]
# Call function_setup with OpenAI model (non-Gemini)
logging_obj, kwargs = function_setup(
original_function="acompletion",
rules_obj=rules_obj,
start_time=datetime.now(),
model="gpt-4",
messages=messages,
litellm_call_id=str(uuid.uuid4()),
custom_llm_provider="openai",
)
# Verify thought signatures were removed
processed_messages = kwargs["messages"]
assert processed_messages[1]["tool_calls"][0]["id"] == "call_123"
assert processed_messages[2]["tool_call_id"] == "call_123"
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"]
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"]
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_thought_signature_preserved_for_gemini():
"""
Test that thought signatures are preserved when sending to Gemini models
"""
rules_obj = Rules()
# Create messages with thought signatures
messages = [
{"role": "user", "content": "What's the weather?"},
{
"role": "assistant",
"tool_calls": [
{
"id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"location": "NYC"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2",
"content": "Rainy, 65°F",
},
]
# Call function_setup with Gemini model
logging_obj, kwargs = function_setup(
original_function="acompletion",
rules_obj=rules_obj,
start_time=datetime.now(),
model="gemini-1.5-pro",
messages=messages,
litellm_call_id=str(uuid.uuid4()),
custom_llm_provider="vertex_ai",
)
# Verify thought signatures were preserved (messages should be unchanged)
processed_messages = kwargs["messages"]
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"]
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"]
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_thought_signature_removal_with_multiple_tool_calls():
"""
Test that thought signatures are removed from multiple tool calls
"""
rules_obj = Rules()
messages = [
{"role": "user", "content": "Get weather and time"},
{
"role": "assistant",
"tool_calls": [
{
"id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1",
"id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}",
"type": "function",
"function": {"name": "get_weather", "arguments": "{}"},
},
{
"id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2",
"type": "function",
"function": {"name": "get_time", "arguments": "{}"},
},
}
for index, call_id in enumerate(call_ids)
],
},
{
"role": "tool",
"tool_call_id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1",
"content": "Sunny",
},
{
"role": "tool",
"tool_call_id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2",
"content": "3:00 PM",
},
*(
{"role": "tool", "tool_call_id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}", "content": "Sunny"}
for index, call_id in enumerate(call_ids)
),
]
logging_obj, kwargs = function_setup(
original_function="acompletion",
rules_obj=rules_obj,
start_time=datetime.now(),
model="claude-3-opus",
messages=messages,
litellm_call_id=str(uuid.uuid4()),
custom_llm_provider="anthropic",
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[("gpt-4", "openai"), ("claude-3-opus", "anthropic"), ("openai/gpt-4", None)],
)
def test_thought_signatures_are_cut_from_every_tool_call_id_for_a_non_gemini_target(model, custom_llm_provider):
stripped = messages_without_foreign_thought_signatures(
"acompletion", model, _signed_tool_turn("call_1", "call_2"), custom_llm_provider
)
processed_messages = kwargs["messages"]
assert stripped is not None
assert [call["id"] for call in stripped[1]["tool_calls"]] == ["call_1", "call_2"]
assert [message["tool_call_id"] for message in stripped[2:]] == ["call_1", "call_2"]
# Check all tool call IDs are cleaned
assert processed_messages[1]["tool_calls"][0]["id"] == "call_1"
assert processed_messages[1]["tool_calls"][1]["id"] == "call_2"
assert processed_messages[2]["tool_call_id"] == "call_1"
assert processed_messages[3]["tool_call_id"] == "call_2"
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[("gemini-1.5-pro", "vertex_ai"), ("vertex_ai/gemini-3.8-flash", None), ("gemini/gemini-3.8-flash", None)],
)
def test_thought_signatures_are_kept_for_a_gemini_target(model, custom_llm_provider):
assert (
messages_without_foreign_thought_signatures(
"acompletion", model, _signed_tool_turn("call_1"), custom_llm_provider
)
is None
)
def test_messages_without_signed_tool_call_ids_are_left_alone_for_a_non_gemini_target():
messages = [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!"}]
assert messages_without_foreign_thought_signatures("acompletion", "gpt-4", messages, "openai") is None
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_messages_without_tool_calls_unchanged():
"""
Test that messages without tool calls pass through unchanged
"""
rules_obj = Rules()
def test_proxy_request_setup_keeps_signatures_for_a_model_name_without_gemini_in_it():
messages = _signed_tool_turn("call_1")
messages = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
logging_obj, kwargs = function_setup(
_, kwargs = function_setup(
original_function="acompletion",
rules_obj=rules_obj,
rules_obj=Rules(),
start_time=datetime.now(),
model="gpt-4",
model="g38flash",
messages=messages,
litellm_call_id=str(uuid.uuid4()),
custom_llm_provider="openai",
)
# Messages should be unchanged
assert kwargs["messages"] == messages
assert kwargs["messages"] is messages, "the proxy alias must not decide signature stripping before routing"
def test_llama_3_prompt():
@ -8990,7 +8894,6 @@ def test_anthropic_messages_pt_interleave_more_thinking_than_tool_groups():
], f"Expected order but got: {types}"
def _paused_web_search_turn(
thinking_blocks: list[dict[str, str]] | None, content: str | list[dict[str, str]] = "Step 1 is Tokyo."
) -> dict[str, object]:
@ -9453,7 +9356,11 @@ TEMPLATE_INITIAL = "<start>"
TEMPLATE_FINAL = "<end>"
PARTIAL_TEMPLATE = {"roles": TEMPLATE_ROLES}
FULL_TEMPLATE = {"roles": TEMPLATE_ROLES, "initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL}
FULL_TEMPLATE = {
"roles": TEMPLATE_ROLES,
"initial_prompt_value": TEMPLATE_INITIAL,
"final_prompt_value": TEMPLATE_FINAL,
}
ROLELESS_TEMPLATE = {"initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL}
@ -9675,7 +9582,9 @@ def test_template_wraps_only_the_roles_it_defines_across_a_conversation(monkeypa
pytest.param({"roles": {"user": {**TEMPLATE_ROLES["user"], "priority": 1}}}, id="extra-role-key"),
pytest.param({"roles": {**TEMPLATE_ROLES, "tool": {"pre_message": 5}}}, id="unused-role-wrong-type"),
pytest.param({"roles": {**TEMPLATE_ROLES, 1: {"pre_message": "x"}}}, id="unused-non-string-role"),
pytest.param({"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys"),
pytest.param(
{"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys"
),
],
)
def test_template_shapes_main_rendered_keep_rendering(template, monkeypatch):