From bda6e9acfc41b014cb7c328a7a6cb3c86005662a Mon Sep 17 00:00:00 2001 From: Felix Mathew Date: Mon, 20 Jul 2026 09:30:26 -0500 Subject: [PATCH] fix(router): address review feedback on Gemini thought signature stripping - Strip the matching tool_call_id on role=tool messages too, not just the assistant's tool_calls[].id, so the two stay paired after a vertex_ai/gemini boundary crossing strips the embedded signature. - Add vertex_ai_beta to the recognized Gemini-family providers, since it is a distinct custom_llm_provider from vertex_ai in this codebase. - Hoist the original model group's provider lookup out of the fallback loop; it does not change across fallback candidates. --- .../router_utils/fallback_event_handlers.py | 13 +++-- litellm/utils.py | 11 +++- .../test_fallback_event_handlers.py | 52 +++++++++++++++++++ tests/test_litellm/test_utils.py | 35 +++++++++++++ 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 9632a121c34..e26d664d4b0 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -19,10 +19,10 @@ else: LitellmRouter = Any -_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini") +_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "vertex_ai_beta", "gemini") -def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]: +def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> str | None: """ Best-effort lookup of the custom_llm_provider for a model group's first deployment. """ @@ -45,7 +45,7 @@ def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_gr return None -def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool: +def _crosses_gemini_endpoint_boundary(original_provider: str | None, fallback_provider: str | None) -> bool: """ True when a fallback switches between Vertex AI and Google AI Studio for the same Gemini model family. Thought signatures minted by one endpoint @@ -160,6 +160,12 @@ async def run_async_fallback( error_from_fallbacks = original_exception fallback_errors = (get_fallback_error_info(original_exception),) + original_messages = kwargs.get("messages") + original_provider = ( + _get_model_group_custom_llm_provider(litellm_router, original_model_group) + if isinstance(original_messages, list) + else None + ) for mg in fallback_model_group: if mg == original_model_group: @@ -178,7 +184,6 @@ async def run_async_fallback( fallback_model_name = kwargs.get("model") messages = kwargs.get("messages") if isinstance(fallback_model_name, str) and isinstance(messages, list): - original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group) fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name) if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider): from litellm.litellm_core_utils.prompt_templates.factory import ( diff --git a/litellm/utils.py b/litellm/utils.py index 5761a14fc0e..673b0eaed3c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -749,13 +749,16 @@ def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_sig def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list: """ - Remove Gemini thought signatures from tool calls in replayed messages. + Remove Gemini thought signatures from tool calls and matching tool + response messages in replayed messages. Thought signatures are endpoint-bound: a signature minted by Vertex AI is rejected as corrupted by Google AI Studio, and vice versa. When the Router falls back across this boundary, the foreign signature must be stripped so the receiving endpoint treats it as missing and re-fills its own dummy - placeholder, instead of replaying an incompatible value. + placeholder, instead of replaying an incompatible value. The assistant + tool call's id and the matching role="tool" message's tool_call_id must + be stripped together, or they no longer match each other. See: https://ai.google.dev/gemini-api/docs/thought-signatures """ @@ -786,6 +789,10 @@ def _remove_foreign_thought_signatures_from_messages(messages: list, thought_sig ) msg_dict["tool_calls"] = new_tool_calls + tool_call_id = msg_dict.get("tool_call_id") + if msg_dict.get("role") == "tool" and isinstance(tool_call_id, str): + msg_dict["tool_call_id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator) + processed_messages.append(msg_dict) return processed_messages diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 45ca20bd3cc..d5315e98d97 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -208,6 +208,13 @@ class TestCrossesGeminiEndpointBoundary: assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True + def test_true_between_vertex_ai_beta_and_gemini(self): + """vertex_ai_beta is a distinct custom_llm_provider from vertex_ai in + this codebase; a deployment configured with it must still be caught + falling back to Google AI Studio.""" + assert _crosses_gemini_endpoint_boundary("vertex_ai_beta", "gemini") is True + assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai_beta") is True + def test_false_when_providers_match(self): assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False @@ -270,3 +277,48 @@ async def test_run_async_fallback_does_not_strip_signature_within_same_provider( tool_call = router.received_kwargs["messages"][0]["tool_calls"][0] assert tool_call["id"] == tool_call_id assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG" + + +class CountingRouter(RecordingRouter): + """Fails on the first fallback candidate and succeeds on the second, so + tests can prove the original model group's provider is only looked up + once even when the loop iterates over multiple fallback candidates.""" + + def __init__(self, deployments_by_model_group=None): + super().__init__(deployments_by_model_group) + self.get_model_list_calls = [] + + def get_model_list(self, model_name=None, team_id=None): + self.get_model_list_calls.append(model_name) + return super().get_model_list(model_name=model_name, team_id=team_id) + + async def async_function_with_fallbacks(self, *args, **kwargs): + if kwargs.get("model") == "fallback-model-1": + raise RuntimeError("first fallback also failed") + self.received_kwargs = kwargs + return StreamingWrapper() + + +@pytest.mark.asyncio +async def test_run_async_fallback_looks_up_original_provider_only_once_across_candidates(): + router = CountingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}], + "fallback-model-1": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}], + "fallback-model-2": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}], + } + ) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model-1", "fallback-model-2"], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + messages=[_gemini_tool_call_message("call_abc123__thought__VERTEX_SIG")], + ) + + assert router.get_model_list_calls.count("primary-model") == 1 + assert router.get_model_list_calls.count("fallback-model-1") == 1 + assert router.get_model_list_calls.count("fallback-model-2") == 1 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 597d8a5ceba..47181020d24 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4937,3 +4937,38 @@ class TestRemoveForeignThoughtSignaturesFromMessages: result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") assert result == messages + + def test_strips_matching_tool_response_id_so_it_still_pairs_with_the_tool_call(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_abc123__thought__VERTEX_SIG", + "content": "72 degrees", + }, + ] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + stripped_tool_call_id = result[0]["tool_calls"][0]["id"] + stripped_tool_response_id = result[1]["tool_call_id"] + assert stripped_tool_call_id == "call_abc123" + assert stripped_tool_response_id == stripped_tool_call_id + + def test_ignores_tool_messages_without_a_signature_suffix(self): + messages = [{"role": "tool", "tool_call_id": "call_abc123", "content": "72 degrees"}] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert result[0]["tool_call_id"] == "call_abc123"