fix(router): address review feedback on Gemini thought signature stripping

- Strip the matching tool_call_id on role=tool messages too, not just
  the assistant's tool_calls[].id, so the two stay paired after a
  vertex_ai/gemini boundary crossing strips the embedded signature.
- Add vertex_ai_beta to the recognized Gemini-family providers, since
  it is a distinct custom_llm_provider from vertex_ai in this codebase.
- Hoist the original model group's provider lookup out of the fallback
  loop; it does not change across fallback candidates.
This commit is contained in:
Felix Mathew 2026-07-20 09:30:26 -05:00
parent afc57c1bed
commit bda6e9acfc
4 changed files with 105 additions and 6 deletions

View file

@ -19,10 +19,10 @@ else:
LitellmRouter = Any
_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini")
_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "vertex_ai_beta", "gemini")
def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]:
def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> str | None:
"""
Best-effort lookup of the custom_llm_provider for a model group's first deployment.
"""
@ -45,7 +45,7 @@ def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_gr
return None
def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool:
def _crosses_gemini_endpoint_boundary(original_provider: str | None, fallback_provider: str | None) -> bool:
"""
True when a fallback switches between Vertex AI and Google AI Studio for
the same Gemini model family. Thought signatures minted by one endpoint
@ -160,6 +160,12 @@ async def run_async_fallback(
error_from_fallbacks = original_exception
fallback_errors = (get_fallback_error_info(original_exception),)
original_messages = kwargs.get("messages")
original_provider = (
_get_model_group_custom_llm_provider(litellm_router, original_model_group)
if isinstance(original_messages, list)
else None
)
for mg in fallback_model_group:
if mg == original_model_group:
@ -178,7 +184,6 @@ async def run_async_fallback(
fallback_model_name = kwargs.get("model")
messages = kwargs.get("messages")
if isinstance(fallback_model_name, str) and isinstance(messages, list):
original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group)
fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name)
if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider):
from litellm.litellm_core_utils.prompt_templates.factory import (

View file

@ -749,13 +749,16 @@ def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_sig
def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list:
"""
Remove Gemini thought signatures from tool calls in replayed messages.
Remove Gemini thought signatures from tool calls and matching tool
response messages in replayed messages.
Thought signatures are endpoint-bound: a signature minted by Vertex AI is
rejected as corrupted by Google AI Studio, and vice versa. When the Router
falls back across this boundary, the foreign signature must be stripped so
the receiving endpoint treats it as missing and re-fills its own dummy
placeholder, instead of replaying an incompatible value.
placeholder, instead of replaying an incompatible value. The assistant
tool call's id and the matching role="tool" message's tool_call_id must
be stripped together, or they no longer match each other.
See: https://ai.google.dev/gemini-api/docs/thought-signatures
"""
@ -786,6 +789,10 @@ def _remove_foreign_thought_signatures_from_messages(messages: list, thought_sig
)
msg_dict["tool_calls"] = new_tool_calls
tool_call_id = msg_dict.get("tool_call_id")
if msg_dict.get("role") == "tool" and isinstance(tool_call_id, str):
msg_dict["tool_call_id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator)
processed_messages.append(msg_dict)
return processed_messages

View file

@ -208,6 +208,13 @@ class TestCrossesGeminiEndpointBoundary:
assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True
assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True
def test_true_between_vertex_ai_beta_and_gemini(self):
"""vertex_ai_beta is a distinct custom_llm_provider from vertex_ai in
this codebase; a deployment configured with it must still be caught
falling back to Google AI Studio."""
assert _crosses_gemini_endpoint_boundary("vertex_ai_beta", "gemini") is True
assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai_beta") is True
def test_false_when_providers_match(self):
assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False
@ -270,3 +277,48 @@ async def test_run_async_fallback_does_not_strip_signature_within_same_provider(
tool_call = router.received_kwargs["messages"][0]["tool_calls"][0]
assert tool_call["id"] == tool_call_id
assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG"
class CountingRouter(RecordingRouter):
"""Fails on the first fallback candidate and succeeds on the second, so
tests can prove the original model group's provider is only looked up
once even when the loop iterates over multiple fallback candidates."""
def __init__(self, deployments_by_model_group=None):
super().__init__(deployments_by_model_group)
self.get_model_list_calls = []
def get_model_list(self, model_name=None, team_id=None):
self.get_model_list_calls.append(model_name)
return super().get_model_list(model_name=model_name, team_id=team_id)
async def async_function_with_fallbacks(self, *args, **kwargs):
if kwargs.get("model") == "fallback-model-1":
raise RuntimeError("first fallback also failed")
self.received_kwargs = kwargs
return StreamingWrapper()
@pytest.mark.asyncio
async def test_run_async_fallback_looks_up_original_provider_only_once_across_candidates():
router = CountingRouter(
deployments_by_model_group={
"primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}],
"fallback-model-1": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}],
"fallback-model-2": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}],
}
)
await run_async_fallback(
litellm_router=router,
fallback_model_group=["fallback-model-1", "fallback-model-2"],
original_model_group="primary-model",
original_exception=RuntimeError("upstream limited request"),
max_fallbacks=3,
fallback_depth=0,
messages=[_gemini_tool_call_message("call_abc123__thought__VERTEX_SIG")],
)
assert router.get_model_list_calls.count("primary-model") == 1
assert router.get_model_list_calls.count("fallback-model-1") == 1
assert router.get_model_list_calls.count("fallback-model-2") == 1

View file

@ -4937,3 +4937,38 @@ class TestRemoveForeignThoughtSignaturesFromMessages:
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
assert result == messages
def test_strips_matching_tool_response_id_so_it_still_pairs_with_the_tool_call(self):
messages = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc123__thought__VERTEX_SIG",
"type": "function",
"provider_specific_fields": {"thought_signature": "VERTEX_SIG"},
"function": {"name": "get_weather", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "call_abc123__thought__VERTEX_SIG",
"content": "72 degrees",
},
]
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
stripped_tool_call_id = result[0]["tool_calls"][0]["id"]
stripped_tool_response_id = result[1]["tool_call_id"]
assert stripped_tool_call_id == "call_abc123"
assert stripped_tool_response_id == stripped_tool_call_id
def test_ignores_tool_messages_without_a_signature_suffix(self):
messages = [{"role": "tool", "tool_call_id": "call_abc123", "content": "72 degrees"}]
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
assert result[0]["tool_call_id"] == "call_abc123"