From afc57c1bed55571058068515cd166df7d08e121e Mon Sep 17 00:00:00 2001 From: Felix Mathew Date: Tue, 7 Jul 2026 14:31:33 -0500 Subject: [PATCH] fix(router): strip foreign Gemini thought signatures on vertex_ai/gemini fallback Thought signatures returned by Gemini tool calls are endpoint-bound: a signature minted by Vertex AI is rejected as corrupted by Google AI Studio, and vice versa. When the Router fell back between a Vertex AI Gemini deployment and its Google AI Studio twin mid-conversation, the foreign signature was replayed unchanged in both the tool call's provider_specific_fields and its ID suffix, causing the fallback call itself to fail with a 400 error. run_async_fallback now detects when a fallback crosses the vertex_ai <-> gemini boundary and strips the foreign signature from replayed tool calls before the fallback request is sent, so the receiving endpoint treats it as missing and fills in its own dummy placeholder, the same way a genuinely missing signature is already handled. Fixes #32140 --- .../router_utils/fallback_event_handlers.py | 54 ++++++++ litellm/utils.py | 63 ++++++++++ .../test_fallback_event_handlers.py | 119 +++++++++++++++++- tests/test_litellm/test_utils.py | 74 +++++++++++ 4 files changed, 309 insertions(+), 1 deletion(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index eb92e9b1c29..9632a121c34 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -19,6 +19,46 @@ else: LitellmRouter = Any +_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini") + + +def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]: + """ + Best-effort lookup of the custom_llm_provider for a model group's first deployment. + """ + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + deployments = litellm_router.get_model_list(model_name=model_group) + if not deployments: + return None + litellm_params = deployments[0].get("litellm_params") or {} + custom_llm_provider = litellm_params.get("custom_llm_provider") + if custom_llm_provider: + return custom_llm_provider + model = litellm_params.get("model") + if not model: + return None + try: + _, inferred_custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + return inferred_custom_llm_provider + except Exception: # noqa: BLE001 # best-effort provider lookup; any failure just skips the boundary check + return None + + +def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool: + """ + True when a fallback switches between Vertex AI and Google AI Studio for + the same Gemini model family. Thought signatures minted by one endpoint + are rejected by the other, so they must be stripped from replayed + messages before the fallback call is made. + """ + return ( + original_provider != fallback_provider + and original_provider in _GEMINI_ENDPOINT_PROVIDERS + and fallback_provider in _GEMINI_ENDPOINT_PROVIDERS + ) + + def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: """ Handles wildcard routing scenario @@ -135,6 +175,20 @@ async def run_async_fallback( kwargs.setdefault("metadata", {}).update( {"model_group": kwargs.get("model", None)} ) # update model_group used, if fallbacks are done + fallback_model_name = kwargs.get("model") + messages = kwargs.get("messages") + if isinstance(fallback_model_name, str) and isinstance(messages, list): + original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group) + fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name) + if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider): + from litellm.litellm_core_utils.prompt_templates.factory import ( + THOUGHT_SIGNATURE_SEPARATOR, + ) + from litellm.utils import _remove_foreign_thought_signatures_from_messages + + kwargs["messages"] = _remove_foreign_thought_signatures_from_messages( + messages, THOUGHT_SIGNATURE_SEPARATOR + ) fallback_depth = fallback_depth + 1 kwargs["fallback_depth"] = fallback_depth kwargs["max_fallbacks"] = max_fallbacks diff --git a/litellm/utils.py b/litellm/utils.py index 174bed09396..5761a14fc0e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -728,6 +728,69 @@ def _remove_thought_signatures_from_messages(messages: List, thought_signature_s return processed_messages +def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_signature_separator: str) -> dict: + """ + Remove a Gemini thought signature from a single tool call dict, both from + provider_specific_fields and from the tool call ID, so the receiving + endpoint treats the signature as missing rather than replaying a foreign one. + """ + tc_dict = tool_call.copy() + + provider_fields = tc_dict.get("provider_specific_fields") + if isinstance(provider_fields, dict) and "thought_signature" in provider_fields: + tc_dict["provider_specific_fields"] = {k: v for k, v in provider_fields.items() if k != "thought_signature"} + + tool_call_id = tc_dict.get("id") + if isinstance(tool_call_id, str): + tc_dict["id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator) + + return tc_dict + + +def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list: + """ + Remove Gemini thought signatures from tool calls in replayed messages. + + Thought signatures are endpoint-bound: a signature minted by Vertex AI is + rejected as corrupted by Google AI Studio, and vice versa. When the Router + falls back across this boundary, the foreign signature must be stripped so + the receiving endpoint treats it as missing and re-fills its own dummy + placeholder, instead of replaying an incompatible value. + + See: https://ai.google.dev/gemini-api/docs/thought-signatures + """ + processed_messages = [] + + for msg in messages: + if hasattr(msg, "model_dump"): + msg_dict = msg.model_dump() + elif isinstance(msg, dict): + msg_dict = msg.copy() + else: + processed_messages.append(msg) + continue + + tool_calls = msg_dict.get("tool_calls") + if msg_dict.get("role") == "assistant" and isinstance(tool_calls, list): + new_tool_calls = [] + for tc in tool_calls: + if hasattr(tc, "model_dump"): + tc_dict = tc.model_dump() + elif isinstance(tc, dict): + tc_dict = tc + else: + new_tool_calls.append(tc) + continue + new_tool_calls.append( + _strip_foreign_thought_signature_from_tool_call(tc_dict, thought_signature_separator) + ) + msg_dict["tool_calls"] = new_tool_calls + + processed_messages.append(msg_dict) + + return processed_messages + + def function_setup( original_function: str, rules_obj, start_time, *args, **kwargs ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 98a34de295c..45ca20bd3cc 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -3,6 +3,8 @@ import json import pytest from litellm.router_utils.fallback_event_handlers import ( + _crosses_gemini_endpoint_boundary, + _get_model_group_custom_llm_provider, get_fallback_model_group, run_async_fallback, ) @@ -84,12 +86,16 @@ async def test_run_async_fallback_raises_when_all_fallbacks_fail(): class RecordingRouter: - def __init__(self): + def __init__(self, deployments_by_model_group=None): self.received_kwargs = None + self.deployments_by_model_group = deployments_by_model_group or {} def log_retry(self, kwargs, e): return kwargs + def get_model_list(self, model_name=None, team_id=None): + return self.deployments_by_model_group.get(model_name) + async def async_function_with_fallbacks(self, *args, **kwargs): self.received_kwargs = kwargs return StreamingWrapper() @@ -153,3 +159,114 @@ def test_get_fallback_model_group_does_not_mutate_fallbacks(): assert fallback_model_group == ["gpt-4o-mini"] assert fallbacks == [{"gpt-3.5-turbo": ["claude-3-haiku"]}, "gpt-4o-mini"] + + +def _gemini_tool_call_message(tool_call_id): + return { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call_id, + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "test_tool", "arguments": "{}"}, + } + ], + } + + +class TestGetModelGroupCustomLlmProvider: + """A thought signature is endpoint-bound, so detecting a fallback's + provider correctly is a prerequisite for knowing whether a boundary + was crossed.""" + + def test_uses_explicit_custom_llm_provider_when_present(self): + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "gemini-3-pro-preview"}}] + } + ) + + assert _get_model_group_custom_llm_provider(router, "primary-model") == "vertex_ai" + + def test_infers_custom_llm_provider_from_model_prefix(self): + router = RecordingRouter( + deployments_by_model_group={"fallback-model": [{"litellm_params": {"model": "gemini/gemini-3-pro-preview"}}]} + ) + + assert _get_model_group_custom_llm_provider(router, "fallback-model") == "gemini" + + def test_returns_none_when_model_group_has_no_deployments(self): + router = RecordingRouter() + + assert _get_model_group_custom_llm_provider(router, "unregistered-model") is None + + +class TestCrossesGeminiEndpointBoundary: + def test_true_between_vertex_ai_and_gemini(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True + assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True + + def test_false_when_providers_match(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False + + def test_false_when_either_side_is_not_a_gemini_endpoint(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "openai") is False + assert _crosses_gemini_endpoint_boundary(None, "gemini") is False + + +@pytest.mark.asyncio +async def test_run_async_fallback_strips_foreign_thought_signature_across_gemini_boundary(): + """The Router falling back from a Vertex AI Gemini deployment to its + Google AI Studio twin (or vice versa) must not replay a thought + signature minted by the other endpoint, or the fallback call itself + gets rejected with a 400 'Corrupted thought signature.' error.""" + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}], + "fallback-model": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}], + } + ) + tool_call_id = "call_abc123__thought__VERTEX_SIG" + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + messages=[_gemini_tool_call_message(tool_call_id)], + ) + + tool_call = router.received_kwargs["messages"][0]["tool_calls"][0] + assert tool_call["id"] == "call_abc123" + assert "thought_signature" not in tool_call["provider_specific_fields"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_strip_signature_within_same_provider(): + """A fallback between two Vertex AI deployments never crosses the + endpoint boundary, so the signature must be replayed unchanged.""" + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}], + "fallback-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}], + } + ) + tool_call_id = "call_abc123__thought__VERTEX_SIG" + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + messages=[_gemini_tool_call_message(tool_call_id)], + ) + + tool_call = router.received_kwargs["messages"][0]["tool_calls"][0] + assert tool_call["id"] == tool_call_id + assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index a1a9448cc58..597d8a5ceba 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -24,6 +24,7 @@ from litellm.utils import ( TextCompletionStreamWrapper, _check_provider_match, _is_streaming_request, + _remove_foreign_thought_signatures_from_messages, get_llm_provider, get_optional_params_image_gen, get_prompt_cache_min_tokens, @@ -4863,3 +4864,76 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192) is False ) + + +class TestRemoveForeignThoughtSignaturesFromMessages: + """A Gemini thought signature minted by one endpoint (Vertex AI or Google + AI Studio) is rejected by the other. These tests cover stripping it from + both channels a replayed tool call can carry it in: the provider_specific_fields + dict and the tool call ID suffix.""" + + def test_strips_signature_from_provider_specific_fields_and_id(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + } + ] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + tool_call = result[0]["tool_calls"][0] + assert tool_call["id"] == "call_abc123" + assert "thought_signature" not in tool_call["provider_specific_fields"] + + def test_leaves_other_provider_specific_fields_untouched(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": { + "thought_signature": "VERTEX_SIG", + "other_field": "keep_me", + }, + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + } + ] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert result[0]["tool_calls"][0]["provider_specific_fields"] == {"other_field": "keep_me"} + + def test_does_not_mutate_the_caller_supplied_messages(self): + original_tool_call = { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "get_weather", "arguments": "{}"}, + } + messages = [{"role": "assistant", "content": None, "tool_calls": [original_tool_call]}] + + _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert original_tool_call["id"] == "call_abc123__thought__VERTEX_SIG" + assert original_tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG" + + def test_ignores_messages_without_tool_calls(self): + messages = [{"role": "user", "content": "hello"}] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert result == messages