fix(router): strip foreign Gemini thought signatures on vertex_ai/gemini fallback

Thought signatures returned by Gemini tool calls are endpoint-bound: a
signature minted by Vertex AI is rejected as corrupted by Google AI
Studio, and vice versa. When the Router fell back between a Vertex AI
Gemini deployment and its Google AI Studio twin mid-conversation, the
foreign signature was replayed unchanged in both the tool call's
provider_specific_fields and its ID suffix, causing the fallback call
itself to fail with a 400 error.

run_async_fallback now detects when a fallback crosses the vertex_ai
<-> gemini boundary and strips the foreign signature from replayed
tool calls before the fallback request is sent, so the receiving
endpoint treats it as missing and fills in its own dummy placeholder,
the same way a genuinely missing signature is already handled.

Fixes #32140
This commit is contained in:
Felix Mathew 2026-07-07 14:31:33 -05:00
parent 067c9bbc96
commit afc57c1bed
4 changed files with 309 additions and 1 deletions

View file

@ -19,6 +19,46 @@ else:
LitellmRouter = Any
_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini")
def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]:
"""
Best-effort lookup of the custom_llm_provider for a model group's first deployment.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
deployments = litellm_router.get_model_list(model_name=model_group)
if not deployments:
return None
litellm_params = deployments[0].get("litellm_params") or {}
custom_llm_provider = litellm_params.get("custom_llm_provider")
if custom_llm_provider:
return custom_llm_provider
model = litellm_params.get("model")
if not model:
return None
try:
_, inferred_custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
return inferred_custom_llm_provider
except Exception: # noqa: BLE001 # best-effort provider lookup; any failure just skips the boundary check
return None
def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool:
"""
True when a fallback switches between Vertex AI and Google AI Studio for
the same Gemini model family. Thought signatures minted by one endpoint
are rejected by the other, so they must be stripped from replayed
messages before the fallback call is made.
"""
return (
original_provider != fallback_provider
and original_provider in _GEMINI_ENDPOINT_PROVIDERS
and fallback_provider in _GEMINI_ENDPOINT_PROVIDERS
)
def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
"""
Handles wildcard routing scenario
@ -135,6 +175,20 @@ async def run_async_fallback(
kwargs.setdefault("metadata", {}).update(
{"model_group": kwargs.get("model", None)}
) # update model_group used, if fallbacks are done
fallback_model_name = kwargs.get("model")
messages = kwargs.get("messages")
if isinstance(fallback_model_name, str) and isinstance(messages, list):
original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group)
fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name)
if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider):
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
)
from litellm.utils import _remove_foreign_thought_signatures_from_messages
kwargs["messages"] = _remove_foreign_thought_signatures_from_messages(
messages, THOUGHT_SIGNATURE_SEPARATOR
)
fallback_depth = fallback_depth + 1
kwargs["fallback_depth"] = fallback_depth
kwargs["max_fallbacks"] = max_fallbacks

View file

@ -728,6 +728,69 @@ def _remove_thought_signatures_from_messages(messages: List, thought_signature_s
return processed_messages
def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_signature_separator: str) -> dict:
"""
Remove a Gemini thought signature from a single tool call dict, both from
provider_specific_fields and from the tool call ID, so the receiving
endpoint treats the signature as missing rather than replaying a foreign one.
"""
tc_dict = tool_call.copy()
provider_fields = tc_dict.get("provider_specific_fields")
if isinstance(provider_fields, dict) and "thought_signature" in provider_fields:
tc_dict["provider_specific_fields"] = {k: v for k, v in provider_fields.items() if k != "thought_signature"}
tool_call_id = tc_dict.get("id")
if isinstance(tool_call_id, str):
tc_dict["id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator)
return tc_dict
def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list:
"""
Remove Gemini thought signatures from tool calls in replayed messages.
Thought signatures are endpoint-bound: a signature minted by Vertex AI is
rejected as corrupted by Google AI Studio, and vice versa. When the Router
falls back across this boundary, the foreign signature must be stripped so
the receiving endpoint treats it as missing and re-fills its own dummy
placeholder, instead of replaying an incompatible value.
See: https://ai.google.dev/gemini-api/docs/thought-signatures
"""
processed_messages = []
for msg in messages:
if hasattr(msg, "model_dump"):
msg_dict = msg.model_dump()
elif isinstance(msg, dict):
msg_dict = msg.copy()
else:
processed_messages.append(msg)
continue
tool_calls = msg_dict.get("tool_calls")
if msg_dict.get("role") == "assistant" and isinstance(tool_calls, list):
new_tool_calls = []
for tc in tool_calls:
if hasattr(tc, "model_dump"):
tc_dict = tc.model_dump()
elif isinstance(tc, dict):
tc_dict = tc
else:
new_tool_calls.append(tc)
continue
new_tool_calls.append(
_strip_foreign_thought_signature_from_tool_call(tc_dict, thought_signature_separator)
)
msg_dict["tool_calls"] = new_tool_calls
processed_messages.append(msg_dict)
return processed_messages
def function_setup(
original_function: str, rules_obj, start_time, *args, **kwargs
): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc.

View file

@ -3,6 +3,8 @@ import json
import pytest
from litellm.router_utils.fallback_event_handlers import (
_crosses_gemini_endpoint_boundary,
_get_model_group_custom_llm_provider,
get_fallback_model_group,
run_async_fallback,
)
@ -84,12 +86,16 @@ async def test_run_async_fallback_raises_when_all_fallbacks_fail():
class RecordingRouter:
def __init__(self):
def __init__(self, deployments_by_model_group=None):
self.received_kwargs = None
self.deployments_by_model_group = deployments_by_model_group or {}
def log_retry(self, kwargs, e):
return kwargs
def get_model_list(self, model_name=None, team_id=None):
return self.deployments_by_model_group.get(model_name)
async def async_function_with_fallbacks(self, *args, **kwargs):
self.received_kwargs = kwargs
return StreamingWrapper()
@ -153,3 +159,114 @@ def test_get_fallback_model_group_does_not_mutate_fallbacks():
assert fallback_model_group == ["gpt-4o-mini"]
assert fallbacks == [{"gpt-3.5-turbo": ["claude-3-haiku"]}, "gpt-4o-mini"]
def _gemini_tool_call_message(tool_call_id):
return {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": tool_call_id,
"type": "function",
"provider_specific_fields": {"thought_signature": "VERTEX_SIG"},
"function": {"name": "test_tool", "arguments": "{}"},
}
],
}
class TestGetModelGroupCustomLlmProvider:
"""A thought signature is endpoint-bound, so detecting a fallback's
provider correctly is a prerequisite for knowing whether a boundary
was crossed."""
def test_uses_explicit_custom_llm_provider_when_present(self):
router = RecordingRouter(
deployments_by_model_group={
"primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "gemini-3-pro-preview"}}]
}
)
assert _get_model_group_custom_llm_provider(router, "primary-model") == "vertex_ai"
def test_infers_custom_llm_provider_from_model_prefix(self):
router = RecordingRouter(
deployments_by_model_group={"fallback-model": [{"litellm_params": {"model": "gemini/gemini-3-pro-preview"}}]}
)
assert _get_model_group_custom_llm_provider(router, "fallback-model") == "gemini"
def test_returns_none_when_model_group_has_no_deployments(self):
router = RecordingRouter()
assert _get_model_group_custom_llm_provider(router, "unregistered-model") is None
class TestCrossesGeminiEndpointBoundary:
def test_true_between_vertex_ai_and_gemini(self):
assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True
assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True
def test_false_when_providers_match(self):
assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False
def test_false_when_either_side_is_not_a_gemini_endpoint(self):
assert _crosses_gemini_endpoint_boundary("vertex_ai", "openai") is False
assert _crosses_gemini_endpoint_boundary(None, "gemini") is False
@pytest.mark.asyncio
async def test_run_async_fallback_strips_foreign_thought_signature_across_gemini_boundary():
"""The Router falling back from a Vertex AI Gemini deployment to its
Google AI Studio twin (or vice versa) must not replay a thought
signature minted by the other endpoint, or the fallback call itself
gets rejected with a 400 'Corrupted thought signature.' error."""
router = RecordingRouter(
deployments_by_model_group={
"primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}],
"fallback-model": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}],
}
)
tool_call_id = "call_abc123__thought__VERTEX_SIG"
await run_async_fallback(
litellm_router=router,
fallback_model_group=["fallback-model"],
original_model_group="primary-model",
original_exception=RuntimeError("upstream limited request"),
max_fallbacks=3,
fallback_depth=0,
messages=[_gemini_tool_call_message(tool_call_id)],
)
tool_call = router.received_kwargs["messages"][0]["tool_calls"][0]
assert tool_call["id"] == "call_abc123"
assert "thought_signature" not in tool_call["provider_specific_fields"]
@pytest.mark.asyncio
async def test_run_async_fallback_does_not_strip_signature_within_same_provider():
"""A fallback between two Vertex AI deployments never crosses the
endpoint boundary, so the signature must be replayed unchanged."""
router = RecordingRouter(
deployments_by_model_group={
"primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}],
"fallback-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}],
}
)
tool_call_id = "call_abc123__thought__VERTEX_SIG"
await run_async_fallback(
litellm_router=router,
fallback_model_group=["fallback-model"],
original_model_group="primary-model",
original_exception=RuntimeError("upstream limited request"),
max_fallbacks=3,
fallback_depth=0,
messages=[_gemini_tool_call_message(tool_call_id)],
)
tool_call = router.received_kwargs["messages"][0]["tool_calls"][0]
assert tool_call["id"] == tool_call_id
assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG"

View file

@ -24,6 +24,7 @@ from litellm.utils import (
TextCompletionStreamWrapper,
_check_provider_match,
_is_streaming_request,
_remove_foreign_thought_signatures_from_messages,
get_llm_provider,
get_optional_params_image_gen,
get_prompt_cache_min_tokens,
@ -4863,3 +4864,76 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model
is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192)
is False
)
class TestRemoveForeignThoughtSignaturesFromMessages:
"""A Gemini thought signature minted by one endpoint (Vertex AI or Google
AI Studio) is rejected by the other. These tests cover stripping it from
both channels a replayed tool call can carry it in: the provider_specific_fields
dict and the tool call ID suffix."""
def test_strips_signature_from_provider_specific_fields_and_id(self):
messages = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc123__thought__VERTEX_SIG",
"type": "function",
"provider_specific_fields": {"thought_signature": "VERTEX_SIG"},
"function": {"name": "get_weather", "arguments": "{}"},
}
],
}
]
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
tool_call = result[0]["tool_calls"][0]
assert tool_call["id"] == "call_abc123"
assert "thought_signature" not in tool_call["provider_specific_fields"]
def test_leaves_other_provider_specific_fields_untouched(self):
messages = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc123__thought__VERTEX_SIG",
"type": "function",
"provider_specific_fields": {
"thought_signature": "VERTEX_SIG",
"other_field": "keep_me",
},
"function": {"name": "get_weather", "arguments": "{}"},
}
],
}
]
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
assert result[0]["tool_calls"][0]["provider_specific_fields"] == {"other_field": "keep_me"}
def test_does_not_mutate_the_caller_supplied_messages(self):
original_tool_call = {
"id": "call_abc123__thought__VERTEX_SIG",
"type": "function",
"provider_specific_fields": {"thought_signature": "VERTEX_SIG"},
"function": {"name": "get_weather", "arguments": "{}"},
}
messages = [{"role": "assistant", "content": None, "tool_calls": [original_tool_call]}]
_remove_foreign_thought_signatures_from_messages(messages, "__thought__")
assert original_tool_call["id"] == "call_abc123__thought__VERTEX_SIG"
assert original_tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG"
def test_ignores_messages_without_tool_calls(self):
messages = [{"role": "user", "content": "hello"}]
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
assert result == messages