test(vertex_ai): cover gemini-3.5-flash and drop assertion-echoing docstrings

Add gemini-3.5-flash to the placeholder-scoping matrix and a regression test
that a natively signed parallel turn replays with no
skip_thought_signature_validator anywhere in the payload, the shape that was
producing empty text responses on 3.5.

Hoist the repeated placeholder expression into one constant and rewrite the
docstrings that restated their own assertions to say why the case matters
instead.
This commit is contained in:
ljogeiger 2026-08-20 00:16:36 +00:00
parent 579291774b
commit a5ad22b8a3

View file

@ -830,13 +830,14 @@ def _parallel_tool_calls_signed_via_id(*signatures):
REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n"
PLACEHOLDER_SIGNATURE = base64.b64encode(b"skip_thought_signature_validator").decode(
"utf-8"
)
def test_dummy_signature_only_on_first_parallel_tool_call():
"""Gemini only returns a thought signature on the first of N parallel function calls.
The sibling calls carry no signature, so replaying them must not fabricate one.
"""
"""Google documents the placeholder as a last resort that degrades quality, so an unsigned
parallel turn replayed to gemini-3 gets a budget of exactly one."""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
@ -850,17 +851,15 @@ def test_dummy_signature_only_on_first_parallel_tool_call():
model="gemini-3-pro-preview",
)
expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode(
"utf-8"
)
assert len(gemini_parts) == 3
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
assert "thoughtSignature" not in gemini_parts[1]
assert "thoughtSignature" not in gemini_parts[2]
def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty():
"""The real signature from Gemini rides on the first call; siblings stay signature-free."""
"""Gemini signs only the first of N parallel function calls, so a faithful replay has
nothing to attach to the siblings."""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
@ -881,7 +880,8 @@ def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty():
def test_real_signature_on_later_parallel_tool_call_is_preserved():
"""A signature attached to a non-first call is still forwarded as-is."""
"""Clients may reorder or drop calls, so a signature that lands on a non-first call is
still the model's own and must survive the round trip."""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
@ -895,11 +895,8 @@ def test_real_signature_on_later_parallel_tool_call_is_preserved():
model="gemini-3-pro-preview",
)
expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode(
"utf-8"
)
assert len(gemini_parts) == 2
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE
@ -970,7 +967,6 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry():
"""A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not
consume the one placeholder slot and leave the real first function call bare."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_get_dummy_thought_signature,
convert_to_gemini_tool_call_invoke,
)
@ -984,7 +980,7 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry():
)
assert len(gemini_parts) == 2
assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature()
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
assert "thoughtSignature" not in gemini_parts[1]
@ -1054,22 +1050,62 @@ def test_parallel_tool_call_history_replayed_through_full_message_conversion():
assert "thoughtSignature" not in model_parts[2]
@pytest.mark.parametrize(
"model",
["gemini-3.5-flash", "vertex_ai/gemini-3.5-flash", "gemini/gemini-3.5-flash"],
)
def test_natively_signed_parallel_turn_never_carries_a_placeholder(model):
"""A native gemini-3.5 parallel turn replays with zero skip_thought_signature_validator parts.
Fabricating the placeholder alongside a real signature is what produced empty text responses
on gemini-3.5 parallel function calling, so the whole payload has to stay placeholder-free.
"""
import json
from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history,
)
messages = [
{"role": "user", "content": "Weather in Paris, London and Tokyo?"},
{
"role": "assistant",
"content": None,
"tool_calls": _parallel_tool_calls_signed_via_id(
REAL_THOUGHT_SIGNATURE, None, None
),
},
]
contents = _gemini_convert_messages_with_history(messages=messages, model=model)
model_parts = contents[1]["parts"]
assert len(model_parts) == 3
assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE
assert "thoughtSignature" not in model_parts[1]
assert "thoughtSignature" not in model_parts[2]
assert PLACEHOLDER_SIGNATURE not in json.dumps(contents)
@pytest.mark.parametrize(
"model",
[
"gemini-3-pro-preview",
"gemini-3-flash-preview",
"gemini-3.1-pro-preview",
"gemini-3.5-flash",
"gemini-3.6-flash",
"gemini-3.7-flash",
"vertex_ai/gemini-3.5-flash",
"vertex_ai/gemini-3.7-flash",
"gemini/gemini-3.5-flash",
"gemini/gemini-3.7-flash",
],
)
def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):
"""Every gemini-3 family member, bare or provider-prefixed, gets one placeholder at most."""
"""The gemini-3 gate is a substring match, so every family member and prefix form has to
land on the same one-placeholder budget rather than only the versions we happened to try."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_get_dummy_thought_signature,
convert_to_gemini_tool_call_invoke,
)
@ -1083,17 +1119,14 @@ def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):
)
assert len(gemini_parts) == 3
assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature()
assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
assert "thoughtSignature" not in gemini_parts[1]
assert "thoughtSignature" not in gemini_parts[2]
def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls():
"""gemini-2.5 history with a signed text part and parallel unsigned calls: the text keeps its real
signature, only the first call gets the placeholder, and the siblings stay bare."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_get_dummy_thought_signature,
)
"""Text-part and function-call signatures are collected by separate code paths, so scoping the
placeholder must not disturb a real signature that arrived on the text part."""
from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history,
)
@ -1111,7 +1144,7 @@ def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls():
assert parts[0]["text"] == "Checking all three cities."
assert parts[0]["thoughtSignature"] == "real_25_signature"
assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature()
assert parts[1]["thoughtSignature"] == PLACEHOLDER_SIGNATURE
assert "thoughtSignature" not in parts[2]
assert "thoughtSignature" not in parts[3]