mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): address review feedback on Gemini thought signature stripping
- Strip the matching tool_call_id on role=tool messages too, not just the assistant's tool_calls[].id, so the two stay paired after a vertex_ai/gemini boundary crossing strips the embedded signature. - Add vertex_ai_beta to the recognized Gemini-family providers, since it is a distinct custom_llm_provider from vertex_ai in this codebase. - Hoist the original model group's provider lookup out of the fallback loop; it does not change across fallback candidates.
This commit is contained in:
parent
afc57c1bed
commit
bda6e9acfc
4 changed files with 105 additions and 6 deletions
|
|
@ -19,10 +19,10 @@ else:
|
|||
LitellmRouter = Any
|
||||
|
||||
|
||||
_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "gemini")
|
||||
_GEMINI_ENDPOINT_PROVIDERS = ("vertex_ai", "vertex_ai_beta", "gemini")
|
||||
|
||||
|
||||
def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> Optional[str]:
|
||||
def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_group: str) -> str | None:
|
||||
"""
|
||||
Best-effort lookup of the custom_llm_provider for a model group's first deployment.
|
||||
"""
|
||||
|
|
@ -45,7 +45,7 @@ def _get_model_group_custom_llm_provider(litellm_router: LitellmRouter, model_gr
|
|||
return None
|
||||
|
||||
|
||||
def _crosses_gemini_endpoint_boundary(original_provider: Optional[str], fallback_provider: Optional[str]) -> bool:
|
||||
def _crosses_gemini_endpoint_boundary(original_provider: str | None, fallback_provider: str | None) -> bool:
|
||||
"""
|
||||
True when a fallback switches between Vertex AI and Google AI Studio for
|
||||
the same Gemini model family. Thought signatures minted by one endpoint
|
||||
|
|
@ -160,6 +160,12 @@ async def run_async_fallback(
|
|||
|
||||
error_from_fallbacks = original_exception
|
||||
fallback_errors = (get_fallback_error_info(original_exception),)
|
||||
original_messages = kwargs.get("messages")
|
||||
original_provider = (
|
||||
_get_model_group_custom_llm_provider(litellm_router, original_model_group)
|
||||
if isinstance(original_messages, list)
|
||||
else None
|
||||
)
|
||||
|
||||
for mg in fallback_model_group:
|
||||
if mg == original_model_group:
|
||||
|
|
@ -178,7 +184,6 @@ async def run_async_fallback(
|
|||
fallback_model_name = kwargs.get("model")
|
||||
messages = kwargs.get("messages")
|
||||
if isinstance(fallback_model_name, str) and isinstance(messages, list):
|
||||
original_provider = _get_model_group_custom_llm_provider(litellm_router, original_model_group)
|
||||
fallback_provider = _get_model_group_custom_llm_provider(litellm_router, fallback_model_name)
|
||||
if _crosses_gemini_endpoint_boundary(original_provider, fallback_provider):
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
|
|
|
|||
|
|
@ -749,13 +749,16 @@ def _strip_foreign_thought_signature_from_tool_call(tool_call: dict, thought_sig
|
|||
|
||||
def _remove_foreign_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list:
|
||||
"""
|
||||
Remove Gemini thought signatures from tool calls in replayed messages.
|
||||
Remove Gemini thought signatures from tool calls and matching tool
|
||||
response messages in replayed messages.
|
||||
|
||||
Thought signatures are endpoint-bound: a signature minted by Vertex AI is
|
||||
rejected as corrupted by Google AI Studio, and vice versa. When the Router
|
||||
falls back across this boundary, the foreign signature must be stripped so
|
||||
the receiving endpoint treats it as missing and re-fills its own dummy
|
||||
placeholder, instead of replaying an incompatible value.
|
||||
placeholder, instead of replaying an incompatible value. The assistant
|
||||
tool call's id and the matching role="tool" message's tool_call_id must
|
||||
be stripped together, or they no longer match each other.
|
||||
|
||||
See: https://ai.google.dev/gemini-api/docs/thought-signatures
|
||||
"""
|
||||
|
|
@ -786,6 +789,10 @@ def _remove_foreign_thought_signatures_from_messages(messages: list, thought_sig
|
|||
)
|
||||
msg_dict["tool_calls"] = new_tool_calls
|
||||
|
||||
tool_call_id = msg_dict.get("tool_call_id")
|
||||
if msg_dict.get("role") == "tool" and isinstance(tool_call_id, str):
|
||||
msg_dict["tool_call_id"] = _remove_thought_signature_from_id(tool_call_id, thought_signature_separator)
|
||||
|
||||
processed_messages.append(msg_dict)
|
||||
|
||||
return processed_messages
|
||||
|
|
|
|||
|
|
@ -208,6 +208,13 @@ class TestCrossesGeminiEndpointBoundary:
|
|||
assert _crosses_gemini_endpoint_boundary("vertex_ai", "gemini") is True
|
||||
assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai") is True
|
||||
|
||||
def test_true_between_vertex_ai_beta_and_gemini(self):
|
||||
"""vertex_ai_beta is a distinct custom_llm_provider from vertex_ai in
|
||||
this codebase; a deployment configured with it must still be caught
|
||||
falling back to Google AI Studio."""
|
||||
assert _crosses_gemini_endpoint_boundary("vertex_ai_beta", "gemini") is True
|
||||
assert _crosses_gemini_endpoint_boundary("gemini", "vertex_ai_beta") is True
|
||||
|
||||
def test_false_when_providers_match(self):
|
||||
assert _crosses_gemini_endpoint_boundary("vertex_ai", "vertex_ai") is False
|
||||
|
||||
|
|
@ -270,3 +277,48 @@ async def test_run_async_fallback_does_not_strip_signature_within_same_provider(
|
|||
tool_call = router.received_kwargs["messages"][0]["tool_calls"][0]
|
||||
assert tool_call["id"] == tool_call_id
|
||||
assert tool_call["provider_specific_fields"]["thought_signature"] == "VERTEX_SIG"
|
||||
|
||||
|
||||
class CountingRouter(RecordingRouter):
|
||||
"""Fails on the first fallback candidate and succeeds on the second, so
|
||||
tests can prove the original model group's provider is only looked up
|
||||
once even when the loop iterates over multiple fallback candidates."""
|
||||
|
||||
def __init__(self, deployments_by_model_group=None):
|
||||
super().__init__(deployments_by_model_group)
|
||||
self.get_model_list_calls = []
|
||||
|
||||
def get_model_list(self, model_name=None, team_id=None):
|
||||
self.get_model_list_calls.append(model_name)
|
||||
return super().get_model_list(model_name=model_name, team_id=team_id)
|
||||
|
||||
async def async_function_with_fallbacks(self, *args, **kwargs):
|
||||
if kwargs.get("model") == "fallback-model-1":
|
||||
raise RuntimeError("first fallback also failed")
|
||||
self.received_kwargs = kwargs
|
||||
return StreamingWrapper()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_looks_up_original_provider_only_once_across_candidates():
|
||||
router = CountingRouter(
|
||||
deployments_by_model_group={
|
||||
"primary-model": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-pro-preview"}}],
|
||||
"fallback-model-1": [{"litellm_params": {"custom_llm_provider": "vertex_ai", "model": "vertex_ai/gemini-3-flash"}}],
|
||||
"fallback-model-2": [{"litellm_params": {"custom_llm_provider": "gemini", "model": "gemini/gemini-3-pro-preview"}}],
|
||||
}
|
||||
)
|
||||
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["fallback-model-1", "fallback-model-2"],
|
||||
original_model_group="primary-model",
|
||||
original_exception=RuntimeError("upstream limited request"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
messages=[_gemini_tool_call_message("call_abc123__thought__VERTEX_SIG")],
|
||||
)
|
||||
|
||||
assert router.get_model_list_calls.count("primary-model") == 1
|
||||
assert router.get_model_list_calls.count("fallback-model-1") == 1
|
||||
assert router.get_model_list_calls.count("fallback-model-2") == 1
|
||||
|
|
|
|||
|
|
@ -4937,3 +4937,38 @@ class TestRemoveForeignThoughtSignaturesFromMessages:
|
|||
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
|
||||
|
||||
assert result == messages
|
||||
|
||||
def test_strips_matching_tool_response_id_so_it_still_pairs_with_the_tool_call(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_abc123__thought__VERTEX_SIG",
|
||||
"type": "function",
|
||||
"provider_specific_fields": {"thought_signature": "VERTEX_SIG"},
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_abc123__thought__VERTEX_SIG",
|
||||
"content": "72 degrees",
|
||||
},
|
||||
]
|
||||
|
||||
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
|
||||
|
||||
stripped_tool_call_id = result[0]["tool_calls"][0]["id"]
|
||||
stripped_tool_response_id = result[1]["tool_call_id"]
|
||||
assert stripped_tool_call_id == "call_abc123"
|
||||
assert stripped_tool_response_id == stripped_tool_call_id
|
||||
|
||||
def test_ignores_tool_messages_without_a_signature_suffix(self):
|
||||
messages = [{"role": "tool", "tool_call_id": "call_abc123", "content": "72 degrees"}]
|
||||
|
||||
result = _remove_foreign_thought_signatures_from_messages(messages, "__thought__")
|
||||
|
||||
assert result[0]["tool_call_id"] == "call_abc123"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue