mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test(vertex_ai): cover gemini-3.5-flash and drop assertion-echoing docstrings
Add gemini-3.5-flash to the placeholder-scoping matrix and a regression test that a natively signed parallel turn replays with no skip_thought_signature_validator anywhere in the payload, the shape that was producing empty text responses on 3.5. Hoist the repeated placeholder expression into one constant and rewrite the docstrings that restated their own assertions to say why the case matters instead.
This commit is contained in:
parent
579291774b
commit
a5ad22b8a3
1 changed files with 58 additions and 25 deletions
|
|
@ -830,13 +830,14 @@ def _parallel_tool_calls_signed_via_id(*signatures):
|
|||
|
||||
|
||||
REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n"
|
||||
PLACEHOLDER_SIGNATURE = base64.b64encode(b"skip_thought_signature_validator").decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_dummy_signature_only_on_first_parallel_tool_call():
|
||||
"""Gemini only returns a thought signature on the first of N parallel function calls.
|
||||
|
||||
The sibling calls carry no signature, so replaying them must not fabricate one.
|
||||
"""
|
||||
"""Google documents the placeholder as a last resort that degrades quality, so an unsigned
|
||||
parallel turn replayed to gemini-3 gets a budget of exactly one."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_gemini_tool_call_invoke,
|
||||
)
|
||||
|
|
@ -850,17 +851,15 @@ def test_dummy_signature_only_on_first_parallel_tool_call():
|
|||
model="gemini-3-pro-preview",
|
||||
)
|
||||
|
||||
expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode(
|
||||
"utf-8"
|
||||
)
|
||||
assert len(gemini_parts) == 3
|
||||
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
|
||||
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
|
||||
assert "thoughtSignature" not in gemini_parts[1]
|
||||
assert "thoughtSignature" not in gemini_parts[2]
|
||||
|
||||
|
||||
def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty():
|
||||
"""The real signature from Gemini rides on the first call; siblings stay signature-free."""
|
||||
"""Gemini signs only the first of N parallel function calls, so a faithful replay has
|
||||
nothing to attach to the siblings."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_gemini_tool_call_invoke,
|
||||
)
|
||||
|
|
@ -881,7 +880,8 @@ def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty():
|
|||
|
||||
|
||||
def test_real_signature_on_later_parallel_tool_call_is_preserved():
|
||||
"""A signature attached to a non-first call is still forwarded as-is."""
|
||||
"""Clients may reorder or drop calls, so a signature that lands on a non-first call is
|
||||
still the model's own and must survive the round trip."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_gemini_tool_call_invoke,
|
||||
)
|
||||
|
|
@ -895,11 +895,8 @@ def test_real_signature_on_later_parallel_tool_call_is_preserved():
|
|||
model="gemini-3-pro-preview",
|
||||
)
|
||||
|
||||
expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode(
|
||||
"utf-8"
|
||||
)
|
||||
assert len(gemini_parts) == 2
|
||||
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
|
||||
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
|
||||
assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE
|
||||
|
||||
|
||||
|
|
@ -970,7 +967,6 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry():
|
|||
"""A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not
|
||||
consume the one placeholder slot and leave the real first function call bare."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
_get_dummy_thought_signature,
|
||||
convert_to_gemini_tool_call_invoke,
|
||||
)
|
||||
|
||||
|
|
@ -984,7 +980,7 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry():
|
|||
)
|
||||
|
||||
assert len(gemini_parts) == 2
|
||||
assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature()
|
||||
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
|
||||
assert "thoughtSignature" not in gemini_parts[1]
|
||||
|
||||
|
||||
|
|
@ -1054,22 +1050,62 @@ def test_parallel_tool_call_history_replayed_through_full_message_conversion():
|
|||
assert "thoughtSignature" not in model_parts[2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["gemini-3.5-flash", "vertex_ai/gemini-3.5-flash", "gemini/gemini-3.5-flash"],
|
||||
)
|
||||
def test_natively_signed_parallel_turn_never_carries_a_placeholder(model):
|
||||
"""A native gemini-3.5 parallel turn replays with zero skip_thought_signature_validator parts.
|
||||
|
||||
Fabricating the placeholder alongside a real signature is what produced empty text responses
|
||||
on gemini-3.5 parallel function calling, so the whole payload has to stay placeholder-free.
|
||||
"""
|
||||
import json
|
||||
|
||||
from litellm.llms.vertex_ai.gemini.transformation import (
|
||||
_gemini_convert_messages_with_history,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Weather in Paris, London and Tokyo?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": _parallel_tool_calls_signed_via_id(
|
||||
REAL_THOUGHT_SIGNATURE, None, None
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
contents = _gemini_convert_messages_with_history(messages=messages, model=model)
|
||||
|
||||
model_parts = contents[1]["parts"]
|
||||
assert len(model_parts) == 3
|
||||
assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE
|
||||
assert "thoughtSignature" not in model_parts[1]
|
||||
assert "thoughtSignature" not in model_parts[2]
|
||||
assert PLACEHOLDER_SIGNATURE not in json.dumps(contents)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"gemini-3-pro-preview",
|
||||
"gemini-3-flash-preview",
|
||||
"gemini-3.1-pro-preview",
|
||||
"gemini-3.5-flash",
|
||||
"gemini-3.6-flash",
|
||||
"gemini-3.7-flash",
|
||||
"vertex_ai/gemini-3.5-flash",
|
||||
"vertex_ai/gemini-3.7-flash",
|
||||
"gemini/gemini-3.5-flash",
|
||||
"gemini/gemini-3.7-flash",
|
||||
],
|
||||
)
|
||||
def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):
|
||||
"""Every gemini-3 family member, bare or provider-prefixed, gets one placeholder at most."""
|
||||
"""The gemini-3 gate is a substring match, so every family member and prefix form has to
|
||||
land on the same one-placeholder budget rather than only the versions we happened to try."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
_get_dummy_thought_signature,
|
||||
convert_to_gemini_tool_call_invoke,
|
||||
)
|
||||
|
||||
|
|
@ -1083,17 +1119,14 @@ def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):
|
|||
)
|
||||
|
||||
assert len(gemini_parts) == 3
|
||||
assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature()
|
||||
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
|
||||
assert "thoughtSignature" not in gemini_parts[1]
|
||||
assert "thoughtSignature" not in gemini_parts[2]
|
||||
|
||||
|
||||
def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls():
|
||||
"""gemini-2.5 history with a signed text part and parallel unsigned calls: the text keeps its real
|
||||
signature, only the first call gets the placeholder, and the siblings stay bare."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
_get_dummy_thought_signature,
|
||||
)
|
||||
"""Text-part and function-call signatures are collected by separate code paths, so scoping the
|
||||
placeholder must not disturb a real signature that arrived on the text part."""
|
||||
from litellm.llms.vertex_ai.gemini.transformation import (
|
||||
_gemini_convert_messages_with_history,
|
||||
)
|
||||
|
|
@ -1111,7 +1144,7 @@ def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls():
|
|||
|
||||
assert parts[0]["text"] == "Checking all three cities."
|
||||
assert parts[0]["thoughtSignature"] == "real_25_signature"
|
||||
assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature()
|
||||
assert parts[1]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
|
||||
assert "thoughtSignature" not in parts[2]
|
||||
assert "thoughtSignature" not in parts[3]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue