From c535022057e05bcbd14a63fd122573e5ad334ed1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:29:09 -0700 Subject: [PATCH] fix(gemini): add thought_signature_in_tool_call_id to keep tool call ids short for clients that cap them --- .../litellm_core_utils/get_litellm_params.py | 1 + .../prompt_templates/factory.py | 4 + .../vertex_and_google_ai_studio_gemini.py | 26 +++- litellm/main.py | 1 + litellm/types/litellm_params.py | 1 + .../coverage_registry/llm_conversational.yaml | 6 +- .../test_gemini_tool_call_id_length_e2e.py | 27 ++-- tests/e2e/models.py | 1 + .../test_thought_signature_in_tool_call_id.py | 145 ++++++++++++++++++ 9 files changed, 194 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..335545bc81b 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -80,6 +80,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_host", "github_copilot_user_session", "fireworks_forward_user_id", + "thought_signature_in_tool_call_id", "responses_api_version", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..d3197972862 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1261,6 +1261,10 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str _encode_tool_call_id_with_signature = encode_tool_call_id_with_signature +def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: + return litellm_params.get("thought_signature_in_tool_call_id") is not False + + def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: """Extract thought signature from tool call's provider_specific_fields. diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 241b2d2a3a0..5001fb0180f 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( + embeds_thought_signature_in_tool_call_id, encode_tool_call_id_with_signature, ) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -1773,6 +1774,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): parts: list[HttpxPartType], cumulative_tool_call_idx: int, is_function_call: bool | None, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[ ChatCompletionToolCallFunctionChunk | None, list[ChatCompletionToolCallChunk] | None, @@ -1811,9 +1813,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # back on the matching `functionResponse`. if gemini_call_id: _tool_response_chunk["id"] = gemini_call_id - # Embed thought signature in ID for OpenAI client compatibility if thought_signature: _tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature} + if thought_signature and thought_signature_in_tool_call_id: _tool_response_chunk["id"] = encode_tool_call_id_with_signature( _tool_response_chunk["id"] or "", thought_signature ) @@ -2473,6 +2475,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model_response: Union[ModelResponse, "ModelResponseStream"], standard_optional_params: dict, cumulative_tool_call_index: int = 0, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[list[dict], list[dict], list, list, int]: """ Helper method to process candidates and extract metadata @@ -2575,6 +2578,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): parts=candidate["content"]["parts"], cumulative_tool_call_idx=cumulative_tool_call_index, is_function_call=is_function_call(standard_optional_params), + thought_signature_in_tool_call_id=thought_signature_in_tool_call_id, ) if "logprobsResult" in candidate: @@ -2658,6 +2662,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model_response: Union[ModelResponse, "ModelResponseStream"], standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract cumulative_tool_call_index: int = 0, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[ # mutable-ok: mirrors override contract list[dict[str, object]], list[dict[str, object]], @@ -2666,7 +2671,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): int, ]: return cls._process_candidates( - _candidates, model_response, standard_optional_params, cumulative_tool_call_index + _candidates, + model_response, + standard_optional_params, + cumulative_tool_call_index, + thought_signature_in_tool_call_id, ) def transform_response( @@ -2749,7 +2758,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): safety_ratings, citation_metadata, _, # cumulative_tool_call_index not needed in non-streaming - ) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params) + ) = VertexGeminiConfig._process_candidates( + _candidates, + model_response, + logging_obj.optional_params, + thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id( + logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ), + ) usage: Final = VertexGeminiConfig._calculate_usage( completion_response=completion_response, @@ -3585,6 +3601,9 @@ class ModelResponseIterator: self.logging_obj = logging_obj self.response_headers = response_headers or {} self.is_function_call = check_is_function_call(logging_obj) + self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id( + logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) self.cumulative_tool_call_index: int = 0 self.has_seen_tool_calls: bool = False self._resume = resume @@ -3649,6 +3668,7 @@ class ModelResponseIterator: model_response, self.logging_obj.optional_params, cumulative_tool_call_index=self.cumulative_tool_call_index, + thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id, ) # Track whether tool_calls have been seen across streaming chunks. diff --git a/litellm/main.py b/litellm/main.py index d647cab24e3..c9641f1cd54 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5992,6 +5992,7 @@ def completion( "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", + "thought_signature_in_tool_call_id", "responses_api_version", ) if key in kwargs diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 5f318cc3c69..11d7b48134a 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -282,6 +282,7 @@ class PromptOptions: @dataclass(frozen=True, slots=True, kw_only=True) class ResponseOptions: merge_reasoning_content_in_choices: bool | None = None + thought_signature_in_tool_call_id: bool | None = None enable_json_schema_validation: bool | None = None complete_response: bool | None = None keepalive_seconds: float | None = None diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 16beed874f8..a73f9c2675a 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -45,9 +45,9 @@ - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} - {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"} - {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"} -- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the downstream 128-character limit"} -- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the downstream 128-character limit"} -- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A Gemini tool-call id remains valid when its thought signature is replayed"} +- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"} +- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"} +- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"} - {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"} - {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"} diff --git a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py index 1203b0a267c..c530822f3a3 100644 --- a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py +++ b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from itertools import chain from typing import Final @@ -25,7 +26,8 @@ pytestmark = pytest.mark.e2e BACKEND_MODEL: Final = "gemini/gemini-3.5-flash" GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY" -MAX_TOOL_CALL_ID_LENGTH: Final = 128 +# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20 +AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") WEATHER_TOOL: Final = ChatTool( function=ChatToolFunction( name="get_weather", @@ -59,7 +61,11 @@ def _register_model(client: PassthroughClient, resources: ResourceManager) -> st model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}" model_id: Final = client.proxy.create_model( model_name, - LiteLLMParamsBody(model=BACKEND_MODEL, api_key=GEMINI_API_KEY), + LiteLLMParamsBody( + model=BACKEND_MODEL, + api_key=GEMINI_API_KEY, + thought_signature_in_tool_call_id=False, + ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) return model_name @@ -94,12 +100,9 @@ def _tool_call_id(call: ToolCall) -> str: return call.id -def _assert_id_lengths(ids: tuple[str, ...]) -> None: - lengths: Final = tuple(len(call_id) for call_id in ids) - assert all(length <= MAX_TOOL_CALL_ID_LENGTH for length in lengths), ( - f"tool call id length(s)={lengths}; each id must be at most " - f"{MAX_TOOL_CALL_ID_LENGTH} characters" - ) +def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None: + rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id)) + assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}" def _stream_tool_calls(events: list[str]) -> tuple[_StreamToolCall, ...]: @@ -113,7 +116,7 @@ class TestGeminiToolCallIdLength: "llm.chat_completions.gemini.tool_use.nonstream.works", exercised_on=["chat_completions"], ) - def test_non_streaming_tool_call_id_fits_128_chars( + def test_non_streaming_tool_call_id_fits_agentcore( self, client: PassthroughClient, resources: ResourceManager ) -> None: model: Final = _register_model(client, resources) @@ -123,13 +126,13 @@ class TestGeminiToolCallIdLength: calls: Final = _tool_calls(response) ids: Final = tuple(_tool_call_id(call) for call in calls) - _assert_id_lengths(ids) + _assert_agentcore_accepts_ids(ids) @pytest.mark.covers( "llm.chat_completions.gemini.tool_use.stream.works", exercised_on=["chat_completions"], ) - def test_streaming_tool_call_id_fits_128_chars( + def test_streaming_tool_call_id_fits_agentcore( self, client: PassthroughClient, resources: ResourceManager ) -> None: model: Final = _register_model(client, resources) @@ -145,7 +148,7 @@ class TestGeminiToolCallIdLength: ids: Final = tuple(call.id for call in calls if call.id) assert ids, f"Gemini stream tool call deltas contained no ids: {calls}" - _assert_id_lengths(ids) + _assert_agentcore_accepts_ids(ids) @pytest.mark.covers( "llm.chat_completions.gemini.multi_turn.nonstream.works", diff --git a/tests/e2e/models.py b/tests/e2e/models.py index afddc7e25d6..a092a37193f 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1363,6 +1363,7 @@ class LiteLLMParamsBody(BaseModel): realtime_protocol: str | None = None allowed_openai_params: list[str] | None = None drop_params: bool | None = None + thought_signature_in_tool_call_id: bool | None = None aws_access_key_id: str | None = Field(default=None, repr=False) aws_secret_access_key: str | None = Field(default=None, repr=False) aws_region_name: str | None = None diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 6c5c3f8c05f..1146e7dac0c 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -9,7 +9,15 @@ Note: Embedding signatures in tool call IDs is a beta feature that requires enable_preview_features=True to be enabled. """ +import json +import re +from collections.abc import Mapping +from itertools import chain +from typing import Final + +import httpx import pytest +import respx import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -285,3 +293,140 @@ def test_parallel_tool_calls_with_signatures(enable_preview_features): assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] sig2 = get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) assert sig2 is None + + +GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*" +BOUNDED_TOOL_CALL_ID: Final = re.compile(r"[A-Za-z0-9_-]{1,64}") +LONG_SIGNATURE: Final = "Ab+/" * 200 +WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} +USER_TURN: Final = {"role": "user", "content": "What is the weather in Paris?"} + + +def _function_call_candidate(native_id: str | None) -> dict[str, object]: + function_call: Final = {"name": "get_weather", "args": {"city": "Paris"}} + return { + "content": { + "role": "model", + "parts": [ + { + "functionCall": function_call if native_id is None else {**function_call, "id": native_id}, + "thoughtSignature": LONG_SIGNATURE, + } + ], + }, + "finishReason": "STOP", + "index": 0, + } + + +def _gemini_body(native_id: str | None) -> dict[str, object]: + return { + "candidates": [_function_call_candidate(native_id)], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 7, "totalTokenCount": 12}, + } + + +def _gemini_response(native_id: str | None, stream: bool) -> httpx.Response: + body: Final = _gemini_body(native_id) + if stream: + return httpx.Response(200, text=f"data: {json.dumps(body)}\n\n", headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=body) + + +def _returned_tool_calls( + native_id: str | None, stream: bool, request_params: Mapping[str, object] +) -> tuple[list[Mapping[str, object]], Mapping[str, object]]: + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(native_id, stream)) + response: Final = litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[USER_TURN], + tools=[WEATHER_TOOL], + stream=stream, + api_key="test-gemini-key", + **request_params, + ) + returned_calls: Final = ( + chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in response) + if stream + else response.choices[0].message.tool_calls + ) + tool_calls: Final = [call.model_dump() for call in returned_calls] + sent_body: Final = json.loads(route.calls.last.request.content) + return tool_calls, sent_body + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("native_id", [None, "kq1z0x7p"]) +def test_tool_call_id_stays_bounded_when_thought_signature_kept_out_of_it(stream: bool, native_id: str | None): + tool_calls, sent_body = _returned_tool_calls(native_id, stream, {"thought_signature_in_tool_call_id": False}) + + tool_call_ids: Final = [call["id"] for call in tool_calls if call["id"]] + assert len(tool_call_ids) == 1 + assert BOUNDED_TOOL_CALL_ID.fullmatch(tool_call_ids[0]) + assert native_id is None or tool_call_ids[0] == native_id + assert tool_calls[0]["provider_specific_fields"]["thought_signature"] == LONG_SIGNATURE + assert "thought_signature_in_tool_call_id" not in json.dumps(sent_body) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_tool_call_id_carries_thought_signature_by_default(stream: bool): + tool_calls, _ = _returned_tool_calls(None, stream, {}) + + tool_call_ids: Final = [call["id"] for call in tool_calls if call["id"]] + assert len(tool_call_ids) == 1 + assert get_thought_signature_from_tool({"id": tool_call_ids[0]}) == LONG_SIGNATURE + + +@pytest.mark.parametrize("client_keeps_provider_fields", [True, False]) +def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields: bool): + tool_calls, _ = _returned_tool_calls(None, False, {"thought_signature_in_tool_call_id": False}) + returned_call: Final = tool_calls[0] + replayed_call: Final = { + "id": returned_call["id"], + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + **( + {"provider_specific_fields": returned_call["provider_specific_fields"]} + if client_keeps_provider_fields + else {} + ), + } + + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + ) + litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[ + USER_TURN, + {"role": "assistant", "content": None, "tool_calls": [replayed_call]}, + {"role": "tool", "tool_call_id": returned_call["id"], "content": '{"temp_c": 21}'}, + ], + tools=[WEATHER_TOOL], + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] + model_parts: Final = chain.from_iterable(turn["parts"] for turn in model_turns) + replayed_signatures: Final = [part["thoughtSignature"] for part in model_parts if "function_call" in part] + assert len(replayed_signatures) == 1 + assert (replayed_signatures[0] == LONG_SIGNATURE) is client_keeps_provider_fields + assert replayed_signatures[0]