diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index eb92e9b1c29..9632a121c34 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -19,6 +19,46 @@ else: LitellmRouter = Any +_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini") + + +def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]: + """ + Best-effort lookup of the custom_llm_provider for a model group's first deployment. + """ + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + deployments = litellm_router.get_model_list(model_name=model_group) + if not deployments: + return None + litellm_params = deployments[0].get("litellm_params") or {} + custom_llm_provider = litellm_params.get("custom_llm_provider") + if custom_llm_provider: + return custom_llm_provider + model = litellm_params.get("model") + if not model: + return None + try: + _, inferred_custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + return inferred_custom_llm_provider + except Exception: # noqa: BLE001 # best-effort provider lookup; any failure just skips the boundary check + return None + + +def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool: + """ + True when a fallback switches between Vertex AI and Google AI Studio for + the same Gemini model family. Thought signatures minted by one endpoint + are rejected by the other, so they must be stripped from replayed + messages before the fallback call is made. + """ + return ( + original_provider != fallback_provider + and original_provider in _GEMINI_ENDPOINT_PROVIDERS + and fallback_provider in _GEMINI_ENDPOINT_PROVIDERS + ) + + def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: """ Handles wildcard routing scenario @@ -135,6 +175,20 @@ async def run_async_fallback( kwargs.setdefault("metadata", {}).update( {"model_group": kwargs.get("model", None)} ) # update model_group used, if fallbacks are done + fallback_model_name = kwargs.get("model") + messages = kwargs.get("messages") + if isinstance(fallback_model_name, str) and isinstance(messages, list): + original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group) + fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name) + if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider): + from litellm.litellm_core_utils.prompt_templates.factory import ( + THOUGHT_SIGNATURE_SEPARATOR, + ) + from litellm.utils import _remove_foreign_thought_signatures_from_messages + + kwargs["messages"] = _remove_foreign_thought_signatures_from_messages( + messages, THOUGHT_SIGNATURE_SEPARATOR + ) fallback_depth = fallback_depth + 1 kwargs["fallback_depth"] = fallback_depth kwargs["max_fallbacks"] = max_fallbacks diff --git a/litellm/utils.py b/litellm/utils.py index 174bed09396..5761a14fc0e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -728,6 +728,69 @@ def _remove_thought_signatures_from_messages(messages: List, thought_signature_s return processed_messages +def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_signature_separator: str) -> dict: + """ + Remove a Gemini thought signature from a single tool call dict, both from + provider_specific_fields and from the tool call ID, so the receiving + endpoint treats the signature as missing rather than replaying a foreign one. + """ + tc_dict = tool_call.copy() + + provider_fields = tc_dict.get("provider_specific_fields") + if isinstance(provider_fields, dict) and "thought_signature" in provider_fields: + tc_dict["provider_specific_fields"] = {k: v for k, v in provider_fields.items() if k != "thought_signature"} + + tool_call_id = tc_dict.get("id") + if isinstance(tool_call_id, str): + tc_dict["id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator) + + return tc_dict + + +def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list: + """ + Remove Gemini thought signatures from tool calls in replayed messages. + + Thought signatures are endpoint-bound: a signature minted by Vertex AI is + rejected as corrupted by Google AI Studio, and vice versa. When the Router + falls back across this boundary, the foreign signature must be stripped so + the receiving endpoint treats it as missing and re-fills its own dummy + placeholder, instead of replaying an incompatible value. + + See: https://ai.google.dev/gemini-api/docs/thought-signatures + """ + processed_messages = [] + + for msg in messages: + if hasattr(msg, "model_dump"): + msg_dict = msg.model_dump() + elif isinstance(msg, dict): + msg_dict = msg.copy() + else: + processed_messages.append(msg) + continue + + tool_calls = msg_dict.get("tool_calls") + if msg_dict.get("role") == "assistant" and isinstance(tool_calls, list): + new_tool_calls = [] + for tc in tool_calls: + if hasattr(tc, "model_dump"): + tc_dict = tc.model_dump() + elif isinstance(tc, dict): + tc_dict = tc + else: + new_tool_calls.append(tc) + continue + new_tool_calls.append( + _strip_foreign_thought_signature_from_tool_call(tc_dict, thought_signature_separator) + ) + msg_dict["tool_calls"] = new_tool_calls + + processed_messages.append(msg_dict) + + return processed_messages + + def function_setup( original_function: str, rules_obj, start_time, *args, **kwargs ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 98a34de295c..45ca20bd3cc 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -3,6 +3,8 @@ import json import pytest from litellm.router_utils.fallback_event_handlers import ( + _crosses_gemini_endpoint_boundary, + _get_model_group_custom_llm_provider, get_fallback_model_group, run_async_fallback, ) @@ -84,12 +86,16 @@ async def test_run_async_fallback_raises_when_all_fallbacks_fail(): class RecordingRouter: - def __init__(self): + def __init__(self, deployments_by_model_group=None): self.received_kwargs = None + self.deployments_by_model_group = deployments_by_model_group or {} def log_retry(self, kwargs, e): return kwargs + def get_model_list(self, model_name=None, team_id=None): + return self.deployments_by_model_group.get(model_name) + async def async_function_with_fallbacks(self, *args, **kwargs): self.received_kwargs = kwargs return StreamingWrapper() @@ -153,3 +159,114 @@ def test_get_fallback_model_group_does_not_mutate_fallbacks(): assert fallback_model_group == ["gpt-4o-mini"] assert fallbacks == [{"gpt-3.5-turbo": ["claude-3-haiku"]}, "gpt-4o-mini"] + + +def _gemini_tool_call_message(tool_call_id): + return { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call_id, + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "test_tool", "arguments": "{}"}, + } + ], + } + + +class TestGetModelGroupCustomLlmProvider: + """A thought signature is endpoint-bound, so detecting a fallback's + provider correctly is a prerequisite for knowing whether a boundary + was crossed.""" + + def test_uses_explicit_custom_llm_provider_when_present(self): + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "gemini-3-pro-preview"}}] + } + ) + + assert _get_model_group_custom_llm_provider(router, "primary-model") == "vertex_ai" + + def test_infers_custom_llm_provider_from_model_prefix(self): + router = RecordingRouter( + deployments_by_model_group={"fallback-model": [{"litellm_params": {"model": "gemini/gemini-3-pro-preview"}}]} + ) + + assert _get_model_group_custom_llm_provider(router, "fallback-model") == "gemini" + + def test_returns_none_when_model_group_has_no_deployments(self): + router = RecordingRouter() + + assert _get_model_group_custom_llm_provider(router, "unregistered-model") is None + + +class TestCrossesGeminiEndpointBoundary: + def test_true_between_vertex_ai_and_gemini(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True + assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True + + def test_false_when_providers_match(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False + + def test_false_when_either_side_is_not_a_gemini_endpoint(self): + assert _crosses_gemini_endpoint_boundary("vertex_ai", "openai") is False + assert _crosses_gemini_endpoint_boundary(None, "gemini") is False + + +@pytest.mark.asyncio +async def test_run_async_fallback_strips_foreign_thought_signature_across_gemini_boundary(): + """The Router falling back from a Vertex AI Gemini deployment to its + Google AI Studio twin (or vice versa) must not replay a thought + signature minted by the other endpoint, or the fallback call itself + gets rejected with a 400 'Corrupted thought signature.' error.""" + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}], + "fallback-model": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}], + } + ) + tool_call_id = "call_abc123__thought__VERTEX_SIG" + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + messages=[_gemini_tool_call_message(tool_call_id)], + ) + + tool_call = router.received_kwargs["messages"][0]["tool_calls"][0] + assert tool_call["id"] == "call_abc123" + assert "thought_signature" not in tool_call["provider_specific_fields"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_strip_signature_within_same_provider(): + """A fallback between two Vertex AI deployments never crosses the + endpoint boundary, so the signature must be replayed unchanged.""" + router = RecordingRouter( + deployments_by_model_group={ + "primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}], + "fallback-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}], + } + ) + tool_call_id = "call_abc123__thought__VERTEX_SIG" + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + messages=[_gemini_tool_call_message(tool_call_id)], + ) + + tool_call = router.received_kwargs["messages"][0]["tool_calls"][0] + assert tool_call["id"] == tool_call_id + assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index a1a9448cc58..597d8a5ceba 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -24,6 +24,7 @@ from litellm.utils import ( TextCompletionStreamWrapper, _check_provider_match, _is_streaming_request, + _remove_foreign_thought_signatures_from_messages, get_llm_provider, get_optional_params_image_gen, get_prompt_cache_min_tokens, @@ -4863,3 +4864,76 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192) is False ) + + +class TestRemoveForeignThoughtSignaturesFromMessages: + """A Gemini thought signature minted by one endpoint (Vertex AI or Google + AI Studio) is rejected by the other. These tests cover stripping it from + both channels a replayed tool call can carry it in: the provider_specific_fields + dict and the tool call ID suffix.""" + + def test_strips_signature_from_provider_specific_fields_and_id(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + } + ] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + tool_call = result[0]["tool_calls"][0] + assert tool_call["id"] == "call_abc123" + assert "thought_signature" not in tool_call["provider_specific_fields"] + + def test_leaves_other_provider_specific_fields_untouched(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": { + "thought_signature": "VERTEX_SIG", + "other_field": "keep_me", + }, + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + } + ] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert result[0]["tool_calls"][0]["provider_specific_fields"] == {"other_field": "keep_me"} + + def test_does_not_mutate_the_caller_supplied_messages(self): + original_tool_call = { + "id": "call_abc123__thought__VERTEX_SIG", + "type": "function", + "provider_specific_fields": {"thought_signature": "VERTEX_SIG"}, + "function": {"name": "get_weather", "arguments": "{}"}, + } + messages = [{"role": "assistant", "content": None, "tool_calls": [original_tool_call]}] + + _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert original_tool_call["id"] == "call_abc123__thought__VERTEX_SIG" + assert original_tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG" + + def test_ignores_messages_without_tool_calls(self): + messages = [{"role": "user", "content": "hello"}] + + result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__") + + assert result == messages