From 2674d983d4c2d3f5cdb72929f827126cf68a1d17 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:09:15 +0000 Subject: [PATCH 01/12] test(e2e): add gemini tool call id length repro for LIT-8912 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../coverage_registry/llm_conversational.yaml | 3 + .../test_gemini_tool_call_id_length_e2e.py | 199 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index cd52e563d69..93de9018ccd 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -41,6 +41,9 @@ - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} - {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"} - {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"} +- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the downstream 128-character limit"} +- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the downstream 128-character limit"} +- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A Gemini tool-call id remains valid when its thought signature is replayed"} - {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"} - {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"} diff --git a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py new file mode 100644 index 00000000000..1203b0a267c --- /dev/null +++ b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from itertools import chain +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import Success, unwrap +from lifecycle import ResourceManager +from models import ( + ChatAssistantTurn, + ChatBody, + ChatMessage, + ChatResponse, + ChatTool, + ChatToolFunction, + ChatToolResultTurn, + LiteLLMParamsBody, + ToolCall, +) +from passthrough_client import PassthroughClient +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + +BACKEND_MODEL: Final = "gemini/gemini-3.5-flash" +GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY" +MAX_TOOL_CALL_ID_LENGTH: Final = 128 +WEATHER_TOOL: Final = ChatTool( + function=ChatToolFunction( + name="get_weather", + description="Get the weather for a city", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + ) +) + + +class _StreamToolCall(BaseModel): + id: str | None = None + + +class _StreamDelta(BaseModel): + tool_calls: tuple[_StreamToolCall, ...] = () + + +class _StreamChoice(BaseModel): + delta: _StreamDelta + + +class _StreamChunk(BaseModel): + choices: tuple[_StreamChoice, ...] = () + + +def _register_model(client: PassthroughClient, resources: ResourceManager) -> str: + model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}" + model_id: Final = client.proxy.create_model( + model_name, + LiteLLMParamsBody(model=BACKEND_MODEL, api_key=GEMINI_API_KEY), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model_name + + +def _tool_request(model: str, *, stream: bool = False) -> ChatBody: + return ChatBody( + model=model, + messages=( + ChatMessage( + role="user", + content=f"Use get_weather for San Francisco and return its result. {unique_marker()}", + ), + ), + tools=(WEATHER_TOOL,), + tool_choice="required", + max_tokens=128, + stream=stream, + ) + + +def _tool_calls(response: ChatResponse) -> tuple[ToolCall, ...]: + assert response.choices, f"Gemini returned no choices: {response}" + message: Final = response.choices[0].message + calls: Final = tuple(message.tool_calls or ()) if message else () + assert calls, f"Gemini returned no tool calls for tool_choice='required': {response}" + return calls + + +def _tool_call_id(call: ToolCall) -> str: + assert call.id, f"Gemini returned a tool call without an id: {call}" + return call.id + + +def _assert_id_lengths(ids: tuple[str, ...]) -> None: + lengths: Final = tuple(len(call_id) for call_id in ids) + assert all(length <= MAX_TOOL_CALL_ID_LENGTH for length in lengths), ( + f"tool call id length(s)={lengths}; each id must be at most " + f"{MAX_TOOL_CALL_ID_LENGTH} characters" + ) + + +def _stream_tool_calls(events: list[str]) -> tuple[_StreamToolCall, ...]: + chunks: Final = tuple(_StreamChunk.model_validate_json(event) for event in events) + choices: Final = chain.from_iterable(chunk.choices for chunk in chunks) + return tuple(chain.from_iterable(choice.delta.tool_calls for choice in choices)) + + +class TestGeminiToolCallIdLength: + @pytest.mark.covers( + "llm.chat_completions.gemini.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_non_streaming_tool_call_id_fits_128_chars( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model: Final = _register_model(client, resources) + response: Final = unwrap( + client.proxy.chat(resources.key(), _tool_request(model)) + ) + calls: Final = _tool_calls(response) + ids: Final = tuple(_tool_call_id(call) for call in calls) + + _assert_id_lengths(ids) + + @pytest.mark.covers( + "llm.chat_completions.gemini.tool_use.stream.works", + exercised_on=["chat_completions"], + ) + def test_streaming_tool_call_id_fits_128_chars( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model: Final = _register_model(client, resources) + result: Final = client.proxy.chat_stream( + resources.key(), _tool_request(model, stream=True) + ) + assert result.ok, f"Gemini chat stream failed: {result}" + assert result.is_streaming, f"Gemini response was not SSE: {result.content_type!r}" + assert result.stream_error is None, f"Gemini stream contained an error: {result.stream_error}" + + calls: Final = _stream_tool_calls(result.stream_events) + assert calls, f"Gemini stream returned no tool call deltas: {result.stream_events[:5]}" + ids: Final = tuple(call.id for call in calls if call.id) + assert ids, f"Gemini stream tool call deltas contained no ids: {calls}" + + _assert_id_lengths(ids) + + @pytest.mark.covers( + "llm.chat_completions.gemini.multi_turn.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_tool_call_round_trip_succeeds( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model: Final = _register_model(client, resources) + key: Final = resources.key() + first_request: Final = _tool_request(model) + first_response: Final = unwrap(client.proxy.chat(key, first_request)) + calls: Final = _tool_calls(first_response) + first_message: Final = first_response.choices[0].message + assert first_message is not None, f"Gemini returned no assistant message: {first_response}" + ids: Final = tuple(_tool_call_id(call) for call in calls) + + second_result: Final = client.proxy.chat( + key, + ChatBody( + model=model, + messages=( + *first_request.messages, + ChatAssistantTurn( + content=first_message.content, + reasoning_content=first_message.reasoning_content, + tool_calls=first_message.tool_calls, + ), + *tuple( + ChatToolResultTurn( + tool_call_id=call_id, + content='{"temperature": 21, "condition": "sunny"}', + ) + for call_id in ids + ), + ), + tools=(WEATHER_TOOL,), + tool_choice="none", + max_tokens=128, + ), + ) + assert isinstance(second_result, Success), f"Gemini turn 2 failed: {second_result}" + assert second_result.status_code == 200, ( + f"Gemini turn 2 returned HTTP {second_result.status_code}, expected 200" + ) + second_response: Final = second_result.data + assert second_response.choices, f"Gemini turn 2 returned no choices: {second_response}" + answer: Final = second_response.choices[0].message + assert answer is not None and answer.content and answer.content.strip(), ( + f"Gemini turn 2 returned an empty answer: {second_response}" + ) From c535022057e05bcbd14a63fd122573e5ad334ed1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:29:09 -0700 Subject: [PATCH 02/12] fix(gemini): add thought_signature_in_tool_call_id to keep tool call ids short for clients that cap them --- .../litellm_core_utils/get_litellm_params.py | 1 + .../prompt_templates/factory.py | 4 + .../vertex_and_google_ai_studio_gemini.py | 26 +++- litellm/main.py | 1 + litellm/types/litellm_params.py | 1 + .../coverage_registry/llm_conversational.yaml | 6 +- .../test_gemini_tool_call_id_length_e2e.py | 27 ++-- tests/e2e/models.py | 1 + .../test_thought_signature_in_tool_call_id.py | 145 ++++++++++++++++++ 9 files changed, 194 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..335545bc81b 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -80,6 +80,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_host", "github_copilot_user_session", "fireworks_forward_user_id", + "thought_signature_in_tool_call_id", "responses_api_version", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..d3197972862 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1261,6 +1261,10 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str _encode_tool_call_id_with_signature = encode_tool_call_id_with_signature +def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: + return litellm_params.get("thought_signature_in_tool_call_id") is not False + + def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: """Extract thought signature from tool call's provider_specific_fields. diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 241b2d2a3a0..5001fb0180f 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( + embeds_thought_signature_in_tool_call_id, encode_tool_call_id_with_signature, ) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -1773,6 +1774,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): parts: list[HttpxPartType], cumulative_tool_call_idx: int, is_function_call: bool | None, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[ ChatCompletionToolCallFunctionChunk | None, list[ChatCompletionToolCallChunk] | None, @@ -1811,9 +1813,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # back on the matching `functionResponse`. if gemini_call_id: _tool_response_chunk["id"] = gemini_call_id - # Embed thought signature in ID for OpenAI client compatibility if thought_signature: _tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature} + if thought_signature and thought_signature_in_tool_call_id: _tool_response_chunk["id"] = encode_tool_call_id_with_signature( _tool_response_chunk["id"] or "", thought_signature ) @@ -2473,6 +2475,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model_response: Union[ModelResponse, "ModelResponseStream"], standard_optional_params: dict, cumulative_tool_call_index: int = 0, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[list[dict], list[dict], list, list, int]: """ Helper method to process candidates and extract metadata @@ -2575,6 +2578,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): parts=candidate["content"]["parts"], cumulative_tool_call_idx=cumulative_tool_call_index, is_function_call=is_function_call(standard_optional_params), + thought_signature_in_tool_call_id=thought_signature_in_tool_call_id, ) if "logprobsResult" in candidate: @@ -2658,6 +2662,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model_response: Union[ModelResponse, "ModelResponseStream"], standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract cumulative_tool_call_index: int = 0, + thought_signature_in_tool_call_id: bool = True, ) -> tuple[ # mutable-ok: mirrors override contract list[dict[str, object]], list[dict[str, object]], @@ -2666,7 +2671,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): int, ]: return cls._process_candidates( - _candidates, model_response, standard_optional_params, cumulative_tool_call_index + _candidates, + model_response, + standard_optional_params, + cumulative_tool_call_index, + thought_signature_in_tool_call_id, ) def transform_response( @@ -2749,7 +2758,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): safety_ratings, citation_metadata, _, # cumulative_tool_call_index not needed in non-streaming - ) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params) + ) = VertexGeminiConfig._process_candidates( + _candidates, + model_response, + logging_obj.optional_params, + thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id( + logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ), + ) usage: Final = VertexGeminiConfig._calculate_usage( completion_response=completion_response, @@ -3585,6 +3601,9 @@ class ModelResponseIterator: self.logging_obj = logging_obj self.response_headers = response_headers or {} self.is_function_call = check_is_function_call(logging_obj) + self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id( + logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) self.cumulative_tool_call_index: int = 0 self.has_seen_tool_calls: bool = False self._resume = resume @@ -3649,6 +3668,7 @@ class ModelResponseIterator: model_response, self.logging_obj.optional_params, cumulative_tool_call_index=self.cumulative_tool_call_index, + thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id, ) # Track whether tool_calls have been seen across streaming chunks. diff --git a/litellm/main.py b/litellm/main.py index d647cab24e3..c9641f1cd54 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5992,6 +5992,7 @@ def completion( "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", + "thought_signature_in_tool_call_id", "responses_api_version", ) if key in kwargs diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 5f318cc3c69..11d7b48134a 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -282,6 +282,7 @@ class PromptOptions: @dataclass(frozen=True, slots=True, kw_only=True) class ResponseOptions: merge_reasoning_content_in_choices: bool | None = None + thought_signature_in_tool_call_id: bool | None = None enable_json_schema_validation: bool | None = None complete_response: bool | None = None keepalive_seconds: float | None = None diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 16beed874f8..a73f9c2675a 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -45,9 +45,9 @@ - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} - {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"} - {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"} -- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the downstream 128-character limit"} -- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the downstream 128-character limit"} -- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A Gemini tool-call id remains valid when its thought signature is replayed"} +- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"} +- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"} +- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"} - {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"} - {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"} diff --git a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py index 1203b0a267c..c530822f3a3 100644 --- a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py +++ b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from itertools import chain from typing import Final @@ -25,7 +26,8 @@ pytestmark = pytest.mark.e2e BACKEND_MODEL: Final = "gemini/gemini-3.5-flash" GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY" -MAX_TOOL_CALL_ID_LENGTH: Final = 128 +# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20 +AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") WEATHER_TOOL: Final = ChatTool( function=ChatToolFunction( name="get_weather", @@ -59,7 +61,11 @@ def _register_model(client: PassthroughClient, resources: ResourceManager) -> st model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}" model_id: Final = client.proxy.create_model( model_name, - LiteLLMParamsBody(model=BACKEND_MODEL, api_key=GEMINI_API_KEY), + LiteLLMParamsBody( + model=BACKEND_MODEL, + api_key=GEMINI_API_KEY, + thought_signature_in_tool_call_id=False, + ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) return model_name @@ -94,12 +100,9 @@ def _tool_call_id(call: ToolCall) -> str: return call.id -def _assert_id_lengths(ids: tuple[str, ...]) -> None: - lengths: Final = tuple(len(call_id) for call_id in ids) - assert all(length <= MAX_TOOL_CALL_ID_LENGTH for length in lengths), ( - f"tool call id length(s)={lengths}; each id must be at most " - f"{MAX_TOOL_CALL_ID_LENGTH} characters" - ) +def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None: + rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id)) + assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}" def _stream_tool_calls(events: list[str]) -> tuple[_StreamToolCall, ...]: @@ -113,7 +116,7 @@ class TestGeminiToolCallIdLength: "llm.chat_completions.gemini.tool_use.nonstream.works", exercised_on=["chat_completions"], ) - def test_non_streaming_tool_call_id_fits_128_chars( + def test_non_streaming_tool_call_id_fits_agentcore( self, client: PassthroughClient, resources: ResourceManager ) -> None: model: Final = _register_model(client, resources) @@ -123,13 +126,13 @@ class TestGeminiToolCallIdLength: calls: Final = _tool_calls(response) ids: Final = tuple(_tool_call_id(call) for call in calls) - _assert_id_lengths(ids) + _assert_agentcore_accepts_ids(ids) @pytest.mark.covers( "llm.chat_completions.gemini.tool_use.stream.works", exercised_on=["chat_completions"], ) - def test_streaming_tool_call_id_fits_128_chars( + def test_streaming_tool_call_id_fits_agentcore( self, client: PassthroughClient, resources: ResourceManager ) -> None: model: Final = _register_model(client, resources) @@ -145,7 +148,7 @@ class TestGeminiToolCallIdLength: ids: Final = tuple(call.id for call in calls if call.id) assert ids, f"Gemini stream tool call deltas contained no ids: {calls}" - _assert_id_lengths(ids) + _assert_agentcore_accepts_ids(ids) @pytest.mark.covers( "llm.chat_completions.gemini.multi_turn.nonstream.works", diff --git a/tests/e2e/models.py b/tests/e2e/models.py index afddc7e25d6..a092a37193f 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1363,6 +1363,7 @@ class LiteLLMParamsBody(BaseModel): realtime_protocol: str | None = None allowed_openai_params: list[str] | None = None drop_params: bool | None = None + thought_signature_in_tool_call_id: bool | None = None aws_access_key_id: str | None = Field(default=None, repr=False) aws_secret_access_key: str | None = Field(default=None, repr=False) aws_region_name: str | None = None diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 6c5c3f8c05f..1146e7dac0c 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -9,7 +9,15 @@ Note: Embedding signatures in tool call IDs is a beta feature that requires enable_preview_features=True to be enabled. """ +import json +import re +from collections.abc import Mapping +from itertools import chain +from typing import Final + +import httpx import pytest +import respx import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -285,3 +293,140 @@ def test_parallel_tool_calls_with_signatures(enable_preview_features): assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] sig2 = get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) assert sig2 is None + + +GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*" +BOUNDED_TOOL_CALL_ID: Final = re.compile(r"[A-Za-z0-9_-]{1,64}") +LONG_SIGNATURE: Final = "Ab+/" * 200 +WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} +USER_TURN: Final = {"role": "user", "content": "What is the weather in Paris?"} + + +def _function_call_candidate(native_id: str | None) -> dict[str, object]: + function_call: Final = {"name": "get_weather", "args": {"city": "Paris"}} + return { + "content": { + "role": "model", + "parts": [ + { + "functionCall": function_call if native_id is None else {**function_call, "id": native_id}, + "thoughtSignature": LONG_SIGNATURE, + } + ], + }, + "finishReason": "STOP", + "index": 0, + } + + +def _gemini_body(native_id: str | None) -> dict[str, object]: + return { + "candidates": [_function_call_candidate(native_id)], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 7, "totalTokenCount": 12}, + } + + +def _gemini_response(native_id: str | None, stream: bool) -> httpx.Response: + body: Final = _gemini_body(native_id) + if stream: + return httpx.Response(200, text=f"data: {json.dumps(body)}\n\n", headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=body) + + +def _returned_tool_calls( + native_id: str | None, stream: bool, request_params: Mapping[str, object] +) -> tuple[list[Mapping[str, object]], Mapping[str, object]]: + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(native_id, stream)) + response: Final = litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[USER_TURN], + tools=[WEATHER_TOOL], + stream=stream, + api_key="test-gemini-key", + **request_params, + ) + returned_calls: Final = ( + chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in response) + if stream + else response.choices[0].message.tool_calls + ) + tool_calls: Final = [call.model_dump() for call in returned_calls] + sent_body: Final = json.loads(route.calls.last.request.content) + return tool_calls, sent_body + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("native_id", [None, "kq1z0x7p"]) +def test_tool_call_id_stays_bounded_when_thought_signature_kept_out_of_it(stream: bool, native_id: str | None): + tool_calls, sent_body = _returned_tool_calls(native_id, stream, {"thought_signature_in_tool_call_id": False}) + + tool_call_ids: Final = [call["id"] for call in tool_calls if call["id"]] + assert len(tool_call_ids) == 1 + assert BOUNDED_TOOL_CALL_ID.fullmatch(tool_call_ids[0]) + assert native_id is None or tool_call_ids[0] == native_id + assert tool_calls[0]["provider_specific_fields"]["thought_signature"] == LONG_SIGNATURE + assert "thought_signature_in_tool_call_id" not in json.dumps(sent_body) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_tool_call_id_carries_thought_signature_by_default(stream: bool): + tool_calls, _ = _returned_tool_calls(None, stream, {}) + + tool_call_ids: Final = [call["id"] for call in tool_calls if call["id"]] + assert len(tool_call_ids) == 1 + assert get_thought_signature_from_tool({"id": tool_call_ids[0]}) == LONG_SIGNATURE + + +@pytest.mark.parametrize("client_keeps_provider_fields", [True, False]) +def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields: bool): + tool_calls, _ = _returned_tool_calls(None, False, {"thought_signature_in_tool_call_id": False}) + returned_call: Final = tool_calls[0] + replayed_call: Final = { + "id": returned_call["id"], + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + **( + {"provider_specific_fields": returned_call["provider_specific_fields"]} + if client_keeps_provider_fields + else {} + ), + } + + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + ) + litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[ + USER_TURN, + {"role": "assistant", "content": None, "tool_calls": [replayed_call]}, + {"role": "tool", "tool_call_id": returned_call["id"], "content": '{"temp_c": 21}'}, + ], + tools=[WEATHER_TOOL], + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] + model_parts: Final = chain.from_iterable(turn["parts"] for turn in model_turns) + replayed_signatures: Final = [part["thoughtSignature"] for part in model_parts if "function_call" in part] + assert len(replayed_signatures) == 1 + assert (replayed_signatures[0] == LONG_SIGNATURE) is client_keeps_provider_fields + assert replayed_signatures[0] From d26b776e5ec42823ee4735f7dfd0d5482b2d854f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:36:41 -0700 Subject: [PATCH 03/12] fix(gemini): pair a tool result with its call when the client re-adds the thought signature to the call id Strands, which the AgentCore harness runs, puts provider_specific_fields.thought_signature back on the assistant tool call id as __thought__ while its tool result keeps the bare id, so the Gemini conversion found no matching call and answered 500. Ids now match with the signature suffix stripped. Also adds thought_signature_in_tool_call_id to the litellm_params inventory test and gives the pass-through tests' mocked logging objects the litellm_params a real Logging object always has. --- .../prompt_templates/factory.py | 15 ++++++- .../test_thought_signature_in_tool_call_id.py | 44 +++++++++++++++++++ ...test_gemini_passthrough_logging_handler.py | 1 + .../test_streaming_handler.py | 1 + tests/unit/types/test_litellm_params.py | 1 + 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d3197972862..ad966fde244 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1261,6 +1261,13 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str _encode_tool_call_id_with_signature = encode_tool_call_id_with_signature +def _same_tool_call(tool_call_id: str, other_tool_call_id: str) -> bool: + return ( + tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] + == other_tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] + ) + + def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: return litellm_params.get("thought_signature_in_tool_call_id") is not False @@ -1609,7 +1616,13 @@ def convert_to_gemini_tool_call_result( msg_tool_call_id: Final = message.get("tool_call_id", None) for tool in tools: prev_tool_call_id = tool.get("id", None) - if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id: + if ( + isinstance(msg_tool_call_id, str) + and isinstance(prev_tool_call_id, str) + and msg_tool_call_id + and prev_tool_call_id + and _same_tool_call(msg_tool_call_id, prev_tool_call_id) + ): name = tool.get("function", {}).get("name", "") # Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix). diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 1146e7dac0c..2df7d0b3dd3 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -430,3 +430,47 @@ def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields assert len(replayed_signatures) == 1 assert (replayed_signatures[0] == LONG_SIGNATURE) is client_keeps_provider_fields assert replayed_signatures[0] + + +def test_tool_result_with_bare_id_matches_call_whose_id_the_client_re_signed(): + bare_id: Final = "call_601401" + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + ) + litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[ + USER_TURN, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": encode_tool_call_id_with_signature(bare_id, LONG_SIGNATURE), + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": bare_id, "content": '{"temp_c": 21}'}, + ], + tools=[WEATHER_TOOL], + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + parts: Final = list(chain.from_iterable(turn["parts"] for turn in sent_body["contents"])) + function_calls: Final = [part for part in parts if "function_call" in part] + function_responses: Final = [part["function_response"] for part in parts if "function_response" in part] + assert [part["thoughtSignature"] for part in function_calls] == [LONG_SIGNATURE] + assert [response["name"] for response in function_responses] == ["get_weather"] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py index f780c056f64..31d95984996 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py @@ -83,6 +83,7 @@ class TestGeminiPassthroughLoggingHandler: mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj) mock_logging_obj.model_call_details = {} mock_logging_obj.optional_params = {} + mock_logging_obj.litellm_params = {} mock_logging_obj.litellm_call_id = "test-call-id-123" return mock_logging_obj diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index da084146894..8bae17ed8b7 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -83,6 +83,7 @@ def _logging_obj() -> LiteLLMLoggingObj: logging_obj = MagicMock(spec=LiteLLMLoggingObj) logging_obj.model_call_details = {} logging_obj.optional_params = {} + logging_obj.litellm_params = {} logging_obj.litellm_call_id = "test-call-id" return logging_obj diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 788d405f99f..d2aed5853ae 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -217,6 +217,7 @@ OPTION_NAMES: Final = ( "assistant_continue_message", "disable_add_transform_inline_image_block", "merge_reasoning_content_in_choices", + "thought_signature_in_tool_call_id", "enable_json_schema_validation", "complete_response", "stream_chunk_size", From f617727ff56c7c2094d27e72829406a6581c9748 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:51:01 -0700 Subject: [PATCH 04/12] fix(gemini): keep the thought signature on streamed Messages and Responses tool calls and accept a string setting --- .../pass_through/adapters/transformation.py | 9 +- .../streaming_iterator.py | 18 ++- litellm/types/router.py | 1 + .../test_thought_signature_in_tool_call_id.py | 106 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 5 files changed, 128 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 9b1297dbd75..29c8bb06c52 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -381,6 +381,10 @@ class LiteLLMAnthropicMessagesAdapter: return None + def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None: + id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1) + return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call) + def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None: """ Extract signature from a tool_use content block's provider_specific_fields. @@ -1658,10 +1662,7 @@ class LiteLLMAnthropicMessagesAdapter: ): raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4()) tool_name = choice.delta.tool_calls[0].function.name or "" - thought_sig: str | None = None - if THOUGHT_SIGNATURE_SEPARATOR in raw_id: - parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1) - thought_sig = parts[1] if len(parts) > 1 else None + thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0]) tool_block: dict[str, object] = { "type": "tool_use", "id": normalize_anthropic_tool_use_id(raw_id), diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 84d4648f75f..cb074c86b38 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,6 +1,6 @@ import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from itertools import filterfalse from typing import Any, Final, cast @@ -82,6 +82,15 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) +def _tool_call_objects(tool_calls: object) -> tuple[object, ...]: + return tuple(tool_calls) if isinstance(tool_calls, list) else () + + +def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]: + fields: Final[object] = getattr(tool_call, "provider_specific_fields", None) + return {"provider_specific_fields": fields} if fields else {} + + def _delta_has_signed_thinking_block(delta: object) -> bool: blocks: Final = getattr(delta, "thinking_blocks", None) or () return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) @@ -404,10 +413,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): except Exception: tool_calls = None - if not tool_calls or not isinstance(tool_calls, list): - return - - for tc in tool_calls: + for tc in _tool_call_objects(tool_calls): call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) if not call_id_raw: continue @@ -482,7 +488,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=output_index, sequence_number=self._sequence_number, - item=BaseLiteLLMOpenAIResponseObject(**item_kwargs), + item=BaseLiteLLMOpenAIResponseObject(**item_kwargs, **_tool_call_provider_specific_fields(tc)), ) self._pending_tool_events.append(item_done_event) diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..ac0b99fe618 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -578,6 +578,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): ) model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) merge_reasoning_content_in_choices: bool | None = False + thought_signature_in_tool_call_id: bool | None = None model_info: dict | None = None mock_response: str | ModelResponse | Exception | object | None = None diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 2df7d0b3dd3..3f64b108a54 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -474,3 +474,109 @@ def test_tool_result_with_bare_id_matches_call_whose_id_the_client_re_signed(): function_responses: Final = [part["function_response"] for part in parts if "function_response" in part] assert [part["thoughtSignature"] for part in function_calls] == [LONG_SIGNATURE] assert [response["name"] for response in function_responses] == ["get_weather"] + + +ANTHROPIC_WEATHER_TOOL: Final = { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, +} + + +def _is_tool_use_start(event: Mapping[str, object]) -> bool: + block: Final = event.get("content_block") + return event.get("type") == "content_block_start" and isinstance(block, dict) and block.get("type") == "tool_use" + + +async def _messages_tool_use_blocks(stream: bool) -> list[Mapping[str, object]]: + with respx.mock: + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, stream)) + response: Final = await litellm.anthropic.messages.acreate( + max_tokens=256, + messages=[USER_TURN], + model="gemini/gemini-3.5-flash", + tools=[ANTHROPIC_WEATHER_TOOL], + stream=stream, + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + if not stream: + return [block for block in response["content"] if block["type"] == "tool_use"] + relayed: Final = b"".join([chunk async for chunk in response]) + events: Final = [json.loads(line.removeprefix(b"data: ")) for line in relayed.splitlines() if line.startswith(b"data:")] + return [event["content_block"] for event in events if _is_tool_use_start(event)] + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id( + stream: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + tool_uses: Final = await _messages_tool_use_blocks(stream) + + assert len(tool_uses) == 1 + assert BOUNDED_TOOL_CALL_ID.fullmatch(tool_uses[0]["id"]) + assert tool_uses[0]["provider_specific_fields"]["signature"] == LONG_SIGNATURE + + +RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_TOOL["function"]} + + +def _responses_function_call_items(stream: bool) -> list[object]: + with respx.mock: + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, stream)) + response: Final = litellm.responses( + model="gemini/gemini-3.5-flash", + input=USER_TURN["content"], + tools=[RESPONSES_WEATHER_TOOL], + stream=stream, + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + items: Final = ( + [event.item for event in response if event.type == "response.output_item.done"] + if stream + else response.output + ) + return [item for item in items if getattr(item, "type", None) == "function_call"] + + +@pytest.mark.parametrize("stream", [False, True]) +def test_responses_function_call_keeps_thought_signature_kept_out_of_its_id(stream: bool): + function_calls: Final = _responses_function_call_items(stream) + + assert len(function_calls) == 1 + assert BOUNDED_TOOL_CALL_ID.fullmatch(function_calls[0].call_id) + assert function_calls[0].provider_specific_fields["thought_signature"] == LONG_SIGNATURE + + +@pytest.mark.parametrize("configured_value", ["false", "False", 0, False]) +def test_deployment_setting_given_as_a_string_or_number_keeps_ids_bounded(configured_value: object): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gemini-agentcore", + "litellm_params": { + "model": "gemini/gemini-3.5-flash", + "api_key": "test-gemini-key", + "thought_signature_in_tool_call_id": configured_value, + }, + } + ] + ) + with respx.mock: + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) + response: Final = router.completion(model="gemini-agentcore", messages=[USER_TURN], tools=[WEATHER_TOOL]) + + assert BOUNDED_TOOL_CALL_ID.fullmatch(response.choices[0].message.tool_calls[0].id) + + +def test_deployment_setting_rejects_a_value_that_is_not_a_boolean(): + with pytest.raises(ValueError, match="thought_signature_in_tool_call_id"): + litellm.Router( + model_list=[ + { + "model_name": "gemini-agentcore", + "litellm_params": {"model": "gemini/gemini-3.5-flash", "thought_signature_in_tool_call_id": "maybe"}, + } + ] + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 244141ec076..f5d5aed7e0d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35587,6 +35587,8 @@ export interface components { tags?: string[] | null; /** Tenant Id */ tenant_id?: string | null; + /** Thought Signature In Tool Call Id */ + thought_signature_in_tool_call_id?: boolean | null; /** Tiered Pricing */ tiered_pricing?: { [key: string]: unknown; @@ -50879,6 +50881,8 @@ export interface components { tags?: string[] | null; /** Tenant Id */ tenant_id?: string | null; + /** Thought Signature In Tool Call Id */ + thought_signature_in_tool_call_id?: boolean | null; /** Tiered Pricing */ tiered_pricing?: { [key: string]: unknown; From 8034fdbb9e9c791f518f39654cdcbffaff99906b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:47:01 -0700 Subject: [PATCH 05/12] fix(gemini): read the thought signature back from Responses input and coerce the per-request setting A Responses function_call input item now keeps its provider_specific_fields, so a client that sends the item back gets its real thought signature replayed to Gemini. A per-request thought_signature_in_tool_call_id of "false" or 0 now keeps ids short, and a value that is not a boolean answers 400, or is dropped under drop_params, instead of a 500. The e2e test now uses the sdk fixture and @meta subjects --- .../prompt_templates/factory.py | 6 +- litellm/main.py | 39 ++- .../transformation.py | 17 +- .../test_gemini_tool_call_id_length_e2e.py | 237 ++++++++---------- .../test_thought_signature_in_tool_call_id.py | 148 ++++++++--- 5 files changed, 275 insertions(+), 172 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ad966fde244..5f5249bfc9f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -221,6 +221,7 @@ def _handle_ollama_system_message(messages: list, prompt: str, msg_i: int) -> tu _OLLAMA_USER_ROLES: Final = frozenset({"user", "tool", "function"}) _DECODED_JSON: Final = TypeAdapter(object) +_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None) def _ollama_bad_message(model: str, message: AllMessageValues, msg_i: int, detail: str) -> "litellm.BadRequestError": @@ -1269,7 +1270,10 @@ def _same_tool_call(tool_call_id: str, other_tool_call_id: str) -> bool: def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: - return litellm_params.get("thought_signature_in_tool_call_id") is not False + try: + return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False + except ValidationError: + return True def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: diff --git a/litellm/main.py b/litellm/main.py index c9641f1cd54..701a16fda32 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -381,6 +381,27 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return assert_never(control) +def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> GenericLiteLLMParams | None: + supplemental: Final = {key: kwargs[key] for key in OPTIONAL_KWARGS_KEYS if key in kwargs} + if not supplemental: + return None + try: + return GenericLiteLLMParams.model_validate(supplemental) + except ValidationError as error: + invalid: Final = tuple(detail for detail in error.errors() if detail["loc"]) + invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid) + if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: + return GenericLiteLLMParams.model_validate( + {key: value for key, value in supplemental.items() if key not in invalid_keys} + ) + raise litellm.BadRequestError( + message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid), + model=model, + llm_provider=None, + body={"param": ", ".join(sorted(invalid_keys))}, + ) from error + + ####### COMPLETION ENDPOINTS ################ @@ -672,17 +693,15 @@ async def acompletion( "enable_json_schema_validation": enable_json_schema_validation, } if custom_llm_provider is None: - _supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects - "dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} - ) + request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs _, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs - "str | None", kwargs.get("api_base") + "str | None", request_kwargs.get("api_base") ) or base_url, - litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None), + litellm_params=_supplemental_provider_params(request_kwargs, model), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -5657,16 +5676,15 @@ def completion( if deployment_id is not None: # azure llms model = deployment_id custom_llm_provider = "azure" - _supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=( - GenericLiteLLMParams.model_validate(_supplemental_provider_params) - if _supplemental_provider_params - else None + litellm_params=_supplemental_provider_params( + routing_kwargs, + cast(str, model), # cast-ok: model is the alias-resolved request model name ), ) @@ -5680,7 +5698,6 @@ def completion( cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg ), ) - routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index d9ebf50cc1a..1c37aaac6fe 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,7 +45,7 @@ from litellm.constants import ( REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -252,8 +252,9 @@ class _ResponsesToolCallItem(Protocol): def get(self, key: str, /) -> object: ... -class _MarkedToolCallChunk(ChatCompletionToolCallChunk): +class _ReplayedToolCallChunk(ChatCompletionToolCallChunk, total=False): cache_control: ReadOnly[ChatCompletionCachedContent] + provider_specific_fields: ReadOnly[Mapping[str, object]] class _MarkedToolMessage(ChatCompletionToolMessage): @@ -2231,7 +2232,7 @@ class LiteLLMCompletionResponsesConfig: tool_call: Final = ( ChatCompletionToolCallChunk(id=tool_call_id, type="function", function=tool_call_function, index=0) if function_call_cache_control is None - else _MarkedToolCallChunk( + else _ReplayedToolCallChunk( id=tool_call_id, type="function", function=tool_call_function, @@ -2239,9 +2240,17 @@ class LiteLLMCompletionResponsesConfig: cache_control=function_call_cache_control, ) ) + provider_specific_fields: Final = as_str_mapping(function_call.get("provider_specific_fields")) + replayed_tool_call: Final = ( + tool_call + if provider_specific_fields is None + else cast( # cast-ok: the same tool call plus the provider_specific_fields the client sent back + _ReplayedToolCallChunk, {**tool_call, "provider_specific_fields": provider_specific_fields} + ) + ) # Create an assistant message with the tool call - return [ChatCompletionResponseMessage(tool_calls=[tool_call], role="assistant", content=None)] + return [ChatCompletionResponseMessage(tool_calls=[replayed_tool_call], role="assistant", content=None)] @staticmethod def _resolve_file_id(item: Mapping[str, object]) -> object: diff --git a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py index c530822f3a3..883b6c08aa2 100644 --- a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py +++ b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py @@ -1,26 +1,27 @@ from __future__ import annotations import re -from itertools import chain +from collections.abc import Iterator from typing import Final import pytest from e2e_config import unique_marker -from e2e_http import Success, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager -from models import ( - ChatAssistantTurn, - ChatBody, - ChatMessage, - ChatResponse, - ChatTool, - ChatToolFunction, - ChatToolResultTurn, - LiteLLMParamsBody, - ToolCall, +from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients +from models import LiteLLMParamsBody +from openai.types.chat import ( + ChatCompletion, + ChatCompletionAssistantMessageParam, + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCallParam, + ChatCompletionToolMessageParam, + ChatCompletionToolParam, + ChatCompletionUserMessageParam, ) +from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall +from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall from passthrough_client import PassthroughClient -from pydantic import BaseModel pytestmark = pytest.mark.e2e @@ -28,33 +29,25 @@ BACKEND_MODEL: Final = "gemini/gemini-3.5-flash" GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY" # bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20 AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") -WEATHER_TOOL: Final = ChatTool( - function=ChatToolFunction( - name="get_weather", - description="Get the weather for a city", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, +WEATHER_TOOL: Final[ChatCompletionToolParam] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} + + +def _subject(mode: Mode) -> Subject: + return Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(BACKEND_MODEL,), + mode=mode, + capabilities=(Capability.FUNCTION_CALLING, Capability.TOOL_CHOICE), ) -) - - -class _StreamToolCall(BaseModel): - id: str | None = None - - -class _StreamDelta(BaseModel): - tool_calls: tuple[_StreamToolCall, ...] = () - - -class _StreamChoice(BaseModel): - delta: _StreamDelta - - -class _StreamChunk(BaseModel): - choices: tuple[_StreamChoice, ...] = () def _register_model(client: PassthroughClient, resources: ResourceManager) -> str: @@ -71,44 +64,42 @@ def _register_model(client: PassthroughClient, resources: ResourceManager) -> st return model_name -def _tool_request(model: str, *, stream: bool = False) -> ChatBody: - return ChatBody( - model=model, - messages=( - ChatMessage( - role="user", - content=f"Use get_weather for San Francisco and return its result. {unique_marker()}", - ), - ), - tools=(WEATHER_TOOL,), - tool_choice="required", - max_tokens=128, - stream=stream, +def _user_message() -> ChatCompletionUserMessageParam: + return {"role": "user", "content": f"Use get_weather for San Francisco and return its result. {unique_marker()}"} + + +def _tool_calls(completion: ChatCompletion) -> tuple[ChatCompletionMessageFunctionToolCall, ...]: + assert completion.choices, f"Gemini returned no choices: {completion.model_dump_json()[:400]}" + calls: Final = tuple( + call + for call in completion.choices[0].message.tool_calls or () + if isinstance(call, ChatCompletionMessageFunctionToolCall) ) - - -def _tool_calls(response: ChatResponse) -> tuple[ToolCall, ...]: - assert response.choices, f"Gemini returned no choices: {response}" - message: Final = response.choices[0].message - calls: Final = tuple(message.tool_calls or ()) if message else () - assert calls, f"Gemini returned no tool calls for tool_choice='required': {response}" + assert calls, f"Gemini returned no tool calls for tool_choice='required': {completion.model_dump_json()[:400]}" return calls -def _tool_call_id(call: ToolCall) -> str: - assert call.id, f"Gemini returned a tool call without an id: {call}" - return call.id - - def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None: rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id)) assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}" -def _stream_tool_calls(events: list[str]) -> tuple[_StreamToolCall, ...]: - chunks: Final = tuple(_StreamChunk.model_validate_json(event) for event in events) - choices: Final = chain.from_iterable(chunk.choices for chunk in chunks) - return tuple(chain.from_iterable(choice.delta.tool_calls for choice in choices)) +def _streamed_tool_calls(chunks: tuple[ChatCompletionChunk, ...]) -> Iterator[ChoiceDeltaToolCall]: + for chunk in chunks: + for choice in chunk.choices: + yield from choice.delta.tool_calls or () + + +def _short_id_tool_call(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionMessageFunctionToolCallParam: + return { + "id": call.id, + "type": "function", + "function": {"name": call.function.name, "arguments": call.function.arguments}, + } + + +def _tool_result(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionToolMessageParam: + return {"role": "tool", "tool_call_id": call.id, "content": '{"temperature": 21, "condition": "sunny"}'} class TestGeminiToolCallIdLength: @@ -116,37 +107,44 @@ class TestGeminiToolCallIdLength: "llm.chat_completions.gemini.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta(_subject(Mode.NONSTREAM)) def test_non_streaming_tool_call_id_fits_agentcore( - self, client: PassthroughClient, resources: ResourceManager + self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients ) -> None: model: Final = _register_model(client, resources) - response: Final = unwrap( - client.proxy.chat(resources.key(), _tool_request(model)) + completion: Final = sdk.openai(resources.key()).chat.completions.create( + model=model, + messages=[_user_message()], + tools=[WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + extra_body=NO_PROXY_CACHE, ) - calls: Final = _tool_calls(response) - ids: Final = tuple(_tool_call_id(call) for call in calls) - _assert_agentcore_accepts_ids(ids) + _assert_agentcore_accepts_ids(tuple(call.id for call in _tool_calls(completion))) @pytest.mark.covers( "llm.chat_completions.gemini.tool_use.stream.works", exercised_on=["chat_completions"], ) + @meta(_subject(Mode.STREAM)) def test_streaming_tool_call_id_fits_agentcore( - self, client: PassthroughClient, resources: ResourceManager + self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients ) -> None: model: Final = _register_model(client, resources) - result: Final = client.proxy.chat_stream( - resources.key(), _tool_request(model, stream=True) + chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple( + sdk.openai(resources.key()).chat.completions.create( + model=model, + messages=[_user_message()], + tools=[WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + stream=True, + extra_body=NO_PROXY_CACHE, + ) ) - assert result.ok, f"Gemini chat stream failed: {result}" - assert result.is_streaming, f"Gemini response was not SSE: {result.content_type!r}" - assert result.stream_error is None, f"Gemini stream contained an error: {result.stream_error}" - - calls: Final = _stream_tool_calls(result.stream_events) - assert calls, f"Gemini stream returned no tool call deltas: {result.stream_events[:5]}" - ids: Final = tuple(call.id for call in calls if call.id) - assert ids, f"Gemini stream tool call deltas contained no ids: {calls}" + ids: Final = tuple(call.id for call in _streamed_tool_calls(chunks) if call.id) + assert ids, f"Gemini stream returned no tool call ids in {len(chunks)} chunks" _assert_agentcore_accepts_ids(ids) @@ -154,49 +152,36 @@ class TestGeminiToolCallIdLength: "llm.chat_completions.gemini.multi_turn.nonstream.works", exercised_on=["chat_completions"], ) + @meta(_subject(Mode.NONSTREAM)) def test_tool_call_round_trip_succeeds( - self, client: PassthroughClient, resources: ResourceManager + self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients ) -> None: model: Final = _register_model(client, resources) - key: Final = resources.key() - first_request: Final = _tool_request(model) - first_response: Final = unwrap(client.proxy.chat(key, first_request)) - calls: Final = _tool_calls(first_response) - first_message: Final = first_response.choices[0].message - assert first_message is not None, f"Gemini returned no assistant message: {first_response}" - ids: Final = tuple(_tool_call_id(call) for call in calls) + openai_client: Final = sdk.openai(resources.key()) + question: Final = _user_message() + first: Final = openai_client.chat.completions.create( + model=model, + messages=[question], + tools=[WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + extra_body=NO_PROXY_CACHE, + ) + calls: Final = _tool_calls(first) + assistant: Final[ChatCompletionAssistantMessageParam] = { + "role": "assistant", + "tool_calls": [_short_id_tool_call(call) for call in calls], + } - second_result: Final = client.proxy.chat( - key, - ChatBody( - model=model, - messages=( - *first_request.messages, - ChatAssistantTurn( - content=first_message.content, - reasoning_content=first_message.reasoning_content, - tool_calls=first_message.tool_calls, - ), - *tuple( - ChatToolResultTurn( - tool_call_id=call_id, - content='{"temperature": 21, "condition": "sunny"}', - ) - for call_id in ids - ), - ), - tools=(WEATHER_TOOL,), - tool_choice="none", - max_tokens=128, - ), - ) - assert isinstance(second_result, Success), f"Gemini turn 2 failed: {second_result}" - assert second_result.status_code == 200, ( - f"Gemini turn 2 returned HTTP {second_result.status_code}, expected 200" - ) - second_response: Final = second_result.data - assert second_response.choices, f"Gemini turn 2 returned no choices: {second_response}" - answer: Final = second_response.choices[0].message - assert answer is not None and answer.content and answer.content.strip(), ( - f"Gemini turn 2 returned an empty answer: {second_response}" + second: Final = openai_client.chat.completions.create( + model=model, + messages=[question, assistant, *(_tool_result(call) for call in calls)], + tools=[WEATHER_TOOL], + tool_choice="none", + max_tokens=128, + extra_body=NO_PROXY_CACHE, ) + + assert second.choices, f"Gemini turn 2 returned no choices: {second.model_dump_json()[:400]}" + answer: Final = second.choices[0].message.content + assert answer and answer.strip(), f"Gemini turn 2 returned an empty answer: {second.model_dump_json()[:400]}" diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 3f64b108a54..6c3aed5b04f 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -11,13 +11,14 @@ enable_preview_features=True to be enabled. import json import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from itertools import chain from typing import Final import httpx import pytest import respx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -339,6 +340,37 @@ def _gemini_response(native_id: str | None, stream: bool) -> httpx.Response: return httpx.Response(200, json=body) +def _gemini_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + + +class _GeminiTurn(TypedDict): + role: ReadOnly[str] + parts: ReadOnly[Sequence[Mapping[str, object]]] + + +class _GeminiRequestBody(TypedDict): + contents: ReadOnly[Sequence[_GeminiTurn]] + + +def _is_bounded(tool_call_id: object) -> bool: + return isinstance(tool_call_id, str) and BOUNDED_TOOL_CALL_ID.fullmatch(tool_call_id) is not None + + +def _replayed_signatures(sent_body: _GeminiRequestBody) -> list[object]: + model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] + model_parts: Final = chain.from_iterable(turn["parts"] for turn in model_turns) + return [part["thoughtSignature"] for part in model_parts if "function_call" in part] + + def _returned_tool_calls( native_id: str | None, stream: bool, request_params: Mapping[str, object] ) -> tuple[list[Mapping[str, object]], Mapping[str, object]]: @@ -369,7 +401,7 @@ def test_tool_call_id_stays_bounded_when_thought_signature_kept_out_of_it(stream tool_call_ids: Final = [call["id"] for call in tool_calls if call["id"]] assert len(tool_call_ids) == 1 - assert BOUNDED_TOOL_CALL_ID.fullmatch(tool_call_ids[0]) + assert _is_bounded(tool_call_ids[0]) assert native_id is None or tool_call_ids[0] == native_id assert tool_calls[0]["provider_specific_fields"]["thought_signature"] == LONG_SIGNATURE assert "thought_signature_in_tool_call_id" not in json.dumps(sent_body) @@ -400,17 +432,7 @@ def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields } with respx.mock: - route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock( - return_value=httpx.Response( - 200, - json={ - "candidates": [ - {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} - ], - "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, - }, - ) - ) + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) litellm.completion( model="gemini/gemini-3.5-flash", messages=[ @@ -424,9 +446,7 @@ def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields ) sent_body: Final = json.loads(route.calls.last.request.content) - model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] - model_parts: Final = chain.from_iterable(turn["parts"] for turn in model_turns) - replayed_signatures: Final = [part["thoughtSignature"] for part in model_parts if "function_call" in part] + replayed_signatures: Final = _replayed_signatures(sent_body) assert len(replayed_signatures) == 1 assert (replayed_signatures[0] == LONG_SIGNATURE) is client_keeps_provider_fields assert replayed_signatures[0] @@ -435,17 +455,7 @@ def test_bounded_tool_call_id_round_trips_to_gemini(client_keeps_provider_fields def test_tool_result_with_bare_id_matches_call_whose_id_the_client_re_signed(): bare_id: Final = "call_601401" with respx.mock: - route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock( - return_value=httpx.Response( - 200, - json={ - "candidates": [ - {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} - ], - "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, - }, - ) - ) + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) litellm.completion( model="gemini/gemini-3.5-flash", messages=[ @@ -514,7 +524,7 @@ async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id( tool_uses: Final = await _messages_tool_use_blocks(stream) assert len(tool_uses) == 1 - assert BOUNDED_TOOL_CALL_ID.fullmatch(tool_uses[0]["id"]) + assert _is_bounded(tool_uses[0]["id"]) assert tool_uses[0]["provider_specific_fields"]["signature"] == LONG_SIGNATURE @@ -545,10 +555,88 @@ def test_responses_function_call_keeps_thought_signature_kept_out_of_its_id(stre function_calls: Final = _responses_function_call_items(stream) assert len(function_calls) == 1 - assert BOUNDED_TOOL_CALL_ID.fullmatch(function_calls[0].call_id) + assert _is_bounded(function_calls[0].call_id) assert function_calls[0].provider_specific_fields["thought_signature"] == LONG_SIGNATURE +@pytest.mark.parametrize("client_keeps_provider_fields", [True, False]) +def test_responses_function_call_replays_its_thought_signature_to_gemini(client_keeps_provider_fields: bool): + function_call: Final = _responses_function_call_items(False)[0] + replayed_item: Final = { + "type": "function_call", + "call_id": function_call.call_id, + "name": function_call.name, + "arguments": function_call.arguments, + **( + {"provider_specific_fields": function_call.provider_specific_fields} + if client_keeps_provider_fields + else {} + ), + } + + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) + litellm.responses( + model="gemini/gemini-3.5-flash", + input=[ + {"role": "user", "content": USER_TURN["content"]}, + replayed_item, + {"type": "function_call_output", "call_id": function_call.call_id, "output": '{"temp_c": 21}'}, + ], + tools=[RESPONSES_WEATHER_TOOL], + api_key="test-gemini-key", + thought_signature_in_tool_call_id=False, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + replayed_signatures: Final = _replayed_signatures(sent_body) + assert len(replayed_signatures) == 1 + assert (replayed_signatures[0] == LONG_SIGNATURE) is client_keeps_provider_fields + assert replayed_signatures[0] + + +@pytest.mark.parametrize("requested_value", ["false", "False", 0]) +def test_request_setting_given_as_a_string_or_number_keeps_ids_bounded(requested_value: object): + tool_calls, _ = _returned_tool_calls(None, False, {"thought_signature_in_tool_call_id": requested_value}) + + assert _is_bounded(tool_calls[0]["id"]) + assert tool_calls[0]["provider_specific_fields"]["thought_signature"] == LONG_SIGNATURE + + +async def _complete(use_async: bool, **request_params: object) -> litellm.ModelResponse: + request: Final = { + "model": "gemini/gemini-3.5-flash", + "messages": [USER_TURN], + "tools": [WEATHER_TOOL], + "api_key": "test-gemini-key", + **request_params, + } + return await litellm.acompletion(**request) if use_async else litellm.completion(**request) + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_request_setting_that_is_not_a_boolean_answers_400(use_async: bool): + with respx.mock(assert_all_called=False): + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) + with pytest.raises(litellm.BadRequestError, match="thought_signature_in_tool_call_id") as raised: + await _complete(use_async, thought_signature_in_tool_call_id="maybe") + + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_request_setting_that_is_not_a_boolean_is_dropped_under_drop_params( + use_async: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) + response: Final = await _complete(use_async, thought_signature_in_tool_call_id="maybe", drop_params=True) + + tool_call_id: Final = response.choices[0].message.tool_calls[0].id + assert get_thought_signature_from_tool({"id": tool_call_id}) == LONG_SIGNATURE + + @pytest.mark.parametrize("configured_value", ["false", "False", 0, False]) def test_deployment_setting_given_as_a_string_or_number_keeps_ids_bounded(configured_value: object): router: Final = litellm.Router( @@ -567,7 +655,7 @@ def test_deployment_setting_given_as_a_string_or_number_keeps_ids_bounded(config respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) response: Final = router.completion(model="gemini-agentcore", messages=[USER_TURN], tools=[WEATHER_TOOL]) - assert BOUNDED_TOOL_CALL_ID.fullmatch(response.choices[0].message.tool_calls[0].id) + assert _is_bounded(response.choices[0].message.tool_calls[0].id) def test_deployment_setting_rejects_a_value_that_is_not_a_boolean(): From 5b3f7ab64c4274a314b390b6ac8de0601b2e0d49 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:01:12 -0700 Subject: [PATCH 06/12] fix(caching): key the response cache on thought_signature_in_tool_call_id --- litellm/caching/caching.py | 13 +++++++++++-- .../litellm_core_utils/model_param_helper.py | 11 +++++++++++ .../prompt_templates/factory.py | 8 -------- .../vertex_and_google_ai_studio_gemini.py | 6 ++---- tests/unit/caching/test_caching.py | 17 +++++++++++++++++ 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..64b7e5b3b7d 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target from litellm._logging import verbose_logger from litellm.constants import CACHED_STREAMING_CHUNK_DELAY from litellm.integrations.otel.runtime import phase_span -from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases +from litellm.litellm_core_utils.model_param_helper import ( + ModelParamHelper, + embeds_thought_signature_in_tool_call_id, + peek_reasoning_summary_aliases, +) from litellm.types.caching import * from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg @@ -419,7 +423,12 @@ class Cache: reasoning_summary_material: Final = ( "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}" ) - hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material) + tool_call_id_shape_material: Final = ( + "" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False" + ) + hashed_cache_key = Cache._get_hashed_cache_key( + cache_key + reasoning_summary_material + tool_call_id_shape_material + ) hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs) verbose_logger.debug( "\nCreated cache key: %s (source material length: %d)", diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index c9c7a0e89ac..b08596e5244 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import ( ResponseCreateParamsNonStreaming, ResponseCreateParamsStreaming, ) +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.anthropic import AnthropicMessagesRequest @@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj return None +_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None) + + +def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: + try: + return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False + except ValidationError: + return True + + class ModelParamHelper: # Cached at class level — deterministic set built from static OpenAI type annotations relevant_logging_args: ClassVar[frozenset[str]] = frozenset() diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5f5249bfc9f..25b7f8109a1 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -221,7 +221,6 @@ def _handle_ollama_system_message(messages: list, prompt: str, msg_i: int) -> tu _OLLAMA_USER_ROLES: Final = frozenset({"user", "tool", "function"}) _DECODED_JSON: Final = TypeAdapter(object) -_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None) def _ollama_bad_message(model: str, message: AllMessageValues, msg_i: int, detail: str) -> "litellm.BadRequestError": @@ -1269,13 +1268,6 @@ def _same_tool_call(tool_call_id: str, other_tool_call_id: str) -> bool: ) -def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool: - try: - return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False - except ValidationError: - return True - - def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: """Extract thought signature from tool call's provider_specific_fields. diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 5001fb0180f..30da28cdb92 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -31,10 +31,8 @@ from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator -from litellm.litellm_core_utils.prompt_templates.factory import ( - embeds_thought_signature_in_tool_call_id, - encode_tool_call_id_with_signature, -) +from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id +from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 91083921130..4ddbf6601d0 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -463,6 +463,23 @@ def test_cache_key_tells_apart_requests_that_differ_only_in_continuation_token() assert cache.get_cache_key(**request, continuation_token="") == first_segment_key +def test_cache_key_tells_apart_requests_that_differ_only_in_tool_call_id_shape() -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL) + request: Final = { + "model": "gemini/gemini-3.5-flash", + "messages": [{"role": "user", "content": "What is the weather in Paris?"}], + } + + signed_ids_key: Final = cache.get_cache_key(**request) + short_ids_key: Final = cache.get_cache_key(**request, thought_signature_in_tool_call_id=False) + + assert short_ids_key != signed_ids_key + assert cache.get_cache_key(**request, thought_signature_in_tool_call_id="false") == short_ids_key + assert cache.get_cache_key(**request, thought_signature_in_tool_call_id=0) == short_ids_key + assert cache.get_cache_key(**request, thought_signature_in_tool_call_id=True) == signed_ids_key + assert cache.get_cache_key(**request, thought_signature_in_tool_call_id=None) == signed_ids_key + + class PhaseRecordingCache(InMemoryCache): """Records the target and the active span each read / write ran under, as a Redis span would.""" From 3f6b36e8bb3d0b5b15ffc299c606c39108b22828 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:32:46 -0700 Subject: [PATCH 07/12] fix(responses): send thought signature fields only with short ids and drop malformed routing kwargs whole --- litellm/main.py | 30 +++++--- .../handler.py | 7 ++ .../session_handler.py | 5 ++ .../streaming_iterator.py | 7 +- .../transformation.py | 37 +++++++--- .../test_thought_signature_in_tool_call_id.py | 60 +++++++++++++++- .../test_litellm_completion_responses.py | 1 + .../test_session_handler.py | 69 +++++++++++++++++++ tests/unit/test_main.py | 61 ++++++++++++++++ 9 files changed, 256 insertions(+), 21 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 701a16fda32..e3b4e5800e4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy +from dataclasses import dataclass from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -381,18 +382,27 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return assert_never(control) -def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> GenericLiteLLMParams | None: +@dataclass(frozen=True, slots=True) +class _SupplementalProviderParams: + params: GenericLiteLLMParams | None + dropped_keys: frozenset[str] = frozenset() + + +def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams: supplemental: Final = {key: kwargs[key] for key in OPTIONAL_KWARGS_KEYS if key in kwargs} if not supplemental: - return None + return _SupplementalProviderParams(params=None) try: - return GenericLiteLLMParams.model_validate(supplemental) + return _SupplementalProviderParams(params=GenericLiteLLMParams.model_validate(supplemental)) except ValidationError as error: invalid: Final = tuple(detail for detail in error.errors() if detail["loc"]) invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid) if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: - return GenericLiteLLMParams.model_validate( - {key: value for key, value in supplemental.items() if key not in invalid_keys} + return _SupplementalProviderParams( + params=GenericLiteLLMParams.model_validate( + {key: value for key, value in supplemental.items() if key not in invalid_keys} + ), + dropped_keys=invalid_keys, ) raise litellm.BadRequestError( message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid), @@ -701,7 +711,7 @@ async def acompletion( "str | None", request_kwargs.get("api_base") ) or base_url, - litellm_params=_supplemental_provider_params(request_kwargs, model), + litellm_params=_supplemental_provider_params(request_kwargs, model).params, ) fallbacks = fallbacks or litellm.model_fallbacks @@ -5446,6 +5456,9 @@ def completion( ### VALIDATE Request ### if model is None: raise ValueError("model param not passed in.") + supplemental_params: Final = _supplemental_provider_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), model) + for dropped_key in supplemental_params.dropped_keys: + kwargs.pop(dropped_key) # validate messages messages = validate_and_fix_openai_messages(messages=messages) tools = validate_and_fix_openai_tools(tools=tools) @@ -5682,10 +5695,7 @@ def completion( custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=_supplemental_provider_params( - routing_kwargs, - cast(str, model), # cast-ok: model is the alias-resolved request model name - ), + litellm_params=supplemental_params.params, ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 5489d2223ac..8a6684917a3 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -8,6 +8,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.responses.additional_tools import hoist_additional_tools from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -99,6 +100,9 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, litellm_metadata=kwargs.get("litellm_metadata", {}), + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id( + _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") @@ -150,5 +154,8 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=responses_api_request, custom_llm_provider=litellm_completion_request.get("custom_llm_provider"), litellm_metadata=kwargs.get("litellm_metadata", {}), + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id( + _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 35160830fdc..34903eda221 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -85,6 +85,7 @@ class ResponsesSessionHandler: previous_response_id: str, item_cache_control: bool = False, client_response_ids: tuple[str, ...] = (), + tool_call_provider_fields: bool = False, ) -> ChatCompletionSession: """ Return the chat completion message history for a previous response id @@ -125,6 +126,7 @@ class ResponsesSessionHandler: proxy_server_request_dict=proxy_server_request_dict, chat_completion_message_history=chat_completion_message_history, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) @@ -150,6 +152,7 @@ class ResponsesSessionHandler: | Message ], item_cache_control: bool = False, + tool_call_provider_fields: bool = False, ): """ Extend the chat completion message history with the spend log payload @@ -182,6 +185,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) @@ -196,6 +200,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) elif isinstance(_response_output, dict) and _response_output and _response_output != {}: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index cb074c86b38..4b41797bb46 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -127,6 +127,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None = None, litellm_metadata: dict | None = None, + tool_call_provider_fields: bool = False, ): self.model: str = model self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper @@ -134,6 +135,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request self.custom_llm_provider: str | None = custom_llm_provider self.litellm_metadata: dict | None = litellm_metadata or {} + self.tool_call_provider_fields: Final = tool_call_provider_fields # Store lightweight dict snapshots for stream_chunk_builder to reduce # repeated Pydantic attribute access in end-of-stream assembly. self.collected_chat_completion_chunks: list[dict[str, object]] = [] @@ -488,7 +490,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=output_index, sequence_number=self._sequence_number, - item=BaseLiteLLMOpenAIResponseObject(**item_kwargs, **_tool_call_provider_specific_fields(tc)), + item=BaseLiteLLMOpenAIResponseObject( + **item_kwargs, + **(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}), + ), ) self._pending_tool_events.append(item_done_event) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 1c37aaac6fe..78c0fae3fe2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -55,6 +55,7 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support @@ -327,17 +328,32 @@ def _dropped_cache_control(_: object) -> None: return None -def _without_cache_control_key(item: object) -> object: +def _without_keys(item: object, keys: frozenset[str]) -> object: if not isinstance(item, dict): return item - return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"} + return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys} + + +def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam: + if not keys or not isinstance(input, list): + return input + return cast( # cast-ok: the same input items, each minus the given keys + ResponseInputParam, [_without_keys(item, keys) for item in input] + ) def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam: - if not isinstance(input, list): - return input - return cast( # cast-ok: the same input items, each minus its own cache_control key - ResponseInputParam, [_without_cache_control_key(item) for item in input] + return _without_item_keys(input, frozenset(("cache_control",))) + + +def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]: + return frozenset( + key + for key, kept in ( + ("cache_control", item_cache_control), + ("provider_specific_fields", tool_call_provider_fields), + ) + if not kept ) @@ -879,6 +895,7 @@ class LiteLLMCompletionResponsesConfig: responses_api_request=responses_api_request, replay_reasoning=True, item_cache_control=keeps_item_marks, + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), ) litellm_completion_request: dict = { @@ -940,6 +957,7 @@ class LiteLLMCompletionResponsesConfig: responses_api_request: ResponsesAPIOptionalRequestParams | dict, replay_reasoning: bool = False, item_cache_control: bool = False, + tool_call_provider_fields: bool = False, ) -> list[ AllMessageValues | GenericChatCompletionMessage @@ -955,7 +973,9 @@ class LiteLLMCompletionResponsesConfig: assistant ``reasoning_content`` and signed ``thinking_blocks`` so the provider gets its own chain-of-thought back instead of reading it as visible text. ``item_cache_control`` belongs to the same callers, when - the target keeps an input item's ``cache_control`` on the message. + the target keeps an input item's ``cache_control`` on the message, and + ``tool_call_provider_fields`` when the target reads a function call's + thought signature from ``provider_specific_fields`` instead of its id. Callers that only inspect the messages (token counting, rate limiting, guardrail scanning) leave both off, because they need every piece of text @@ -978,7 +998,7 @@ class LiteLLMCompletionResponsesConfig: messages.extend( LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=input if item_cache_control else without_item_cache_control(input), + input=_without_item_keys(input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)), replay_reasoning=replay_reasoning, ) ) @@ -1021,6 +1041,7 @@ class LiteLLMCompletionResponsesConfig: ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, item_cache_control=keeps_item_marks, client_response_ids=previous_response_client_ids, + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), ) chat_completion_session: Final = ( await lookup(previous_response_id) diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 6c3aed5b04f..77682722dfa 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -13,6 +13,7 @@ import json import re from collections.abc import Mapping, Sequence from itertools import chain +from types import MappingProxyType from typing import Final import httpx @@ -529,9 +530,10 @@ async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id( RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_TOOL["function"]} +SHORT_IDS: Final = MappingProxyType({"thought_signature_in_tool_call_id": False}) -def _responses_function_call_items(stream: bool) -> list[object]: +def _responses_function_call_items(stream: bool, request_params: Mapping[str, object] = SHORT_IDS) -> list[object]: with respx.mock: respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, stream)) response: Final = litellm.responses( @@ -540,7 +542,7 @@ def _responses_function_call_items(stream: bool) -> list[object]: tools=[RESPONSES_WEATHER_TOOL], stream=stream, api_key="test-gemini-key", - thought_signature_in_tool_call_id=False, + **request_params, ) items: Final = ( [event.item for event in response if event.type == "response.output_item.done"] @@ -559,6 +561,60 @@ def test_responses_function_call_keeps_thought_signature_kept_out_of_its_id(stre assert function_calls[0].provider_specific_fields["thought_signature"] == LONG_SIGNATURE +def test_streamed_function_call_carries_its_thought_signature_only_in_its_id_by_default(): + function_calls: Final = _responses_function_call_items(True, {}) + + assert len(function_calls) == 1 + assert get_thought_signature_from_tool({"id": function_calls[0].call_id}) == LONG_SIGNATURE + assert getattr(function_calls[0], "provider_specific_fields", None) is None + + +def _deepseek_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "deepseek-v4-flash", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sunny"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + + +@pytest.mark.parametrize("request_params", [{}, SHORT_IDS]) +def test_responses_function_call_provider_fields_reach_the_provider_only_with_short_ids( + request_params: Mapping[str, object], +): + replayed_item: Final = { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "provider_specific_fields": {"thought_signature": LONG_SIGNATURE}, + } + + with respx.mock: + route: Final = respx.post(url__regex=r"https://api\.deepseek\.com/.*").mock(return_value=_deepseek_answer()) + litellm.responses( + model="deepseek/deepseek-v4-flash", + input=[ + {"role": "user", "content": USER_TURN["content"]}, + replayed_item, + {"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'}, + ], + tools=[RESPONSES_WEATHER_TOOL], + api_key="test-deepseek-key", + **request_params, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + sent_tool_calls: Final = list(chain.from_iterable(message.get("tool_calls") or () for message in sent_body["messages"])) + assert len(sent_tool_calls) == 1 + assert ("provider_specific_fields" in sent_tool_calls[0]) is bool(request_params) + + @pytest.mark.parametrize("client_keeps_provider_fields", [True, False]) def test_responses_function_call_replays_its_thought_signature_to_gemini(client_keeps_provider_fields: bool): function_call: Final = _responses_function_call_items(False)[0] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f34ca7d0ff8..674c65fb130 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4318,6 +4318,7 @@ class TestEnsureOutputItemContentPartAdded: iterator._queued_web_search_call_ids = set() iterator._tool_search_requested = False iterator._tool_search_call_ids = set() + iterator.tool_call_provider_fields = False return iterator def _make_text_chunk(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 7a146c1b665..26823ba9534 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1,11 +1,14 @@ import json import sqlite3 from collections.abc import Sequence +from itertools import chain from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, patch import pytest +import httpx +import respx from fastapi import HTTPException from fastapi.testclient import TestClient @@ -955,3 +958,69 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_ assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"] assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"] + + +GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*" +LONG_SIGNATURE: Final = "Ab+/" * 200 + + +def _gemini_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + + +def _stored_function_call_turn(response_id: str) -> dict: + return { + "request_id": response_id, + "call_type": "aresponses", + "session_id": "gemini-session", + "proxy_server_request": {"input": "What is the weather in Paris?", "model": "gemini-short-ids"}, + "response": { + "id": response_id, + "object": "response", + "output": [ + { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + "provider_specific_fields": {"thought_signature": LONG_SIGNATURE}, + } + ], + }, + } + + +@pytest.mark.parametrize("short_ids", [False, True]) +async def test_previous_response_replays_a_thought_signature_kept_out_of_the_id_only_with_short_ids( + short_ids: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="gemini", model_id=None, response_id="chatcmpl-gemini-turn-1" + ) + stored: Final = _StoredSpendLogs((_stored_function_call_turn(response_id),)) + + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored)), respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) + await litellm.aresponses( + model="gemini/gemini-3.5-flash", + input=[{"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'}], + previous_response_id=response_id, + api_key="test-gemini-key", + **({"thought_signature_in_tool_call_id": False} if short_ids else {}), + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] + model_parts: Final = list(chain.from_iterable(turn["parts"] for turn in model_turns)) + assert len(model_parts) == 1 + assert (model_parts[0]["thoughtSignature"] == LONG_SIGNATURE) is short_ids diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f5d13aecb6f..d6f97692eec 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -9810,3 +9810,64 @@ async def test_converse_fallback_prices_the_current_attempt( else: client.close() configuration.reset_rust_configuration() + + +def _openai_chat_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +async def _openai_completion(use_async: bool, **request_params: object) -> litellm.ModelResponse: + request: Final = { + "model": "openai/gpt-5.6", + "messages": [{"role": "user", "content": "hi"}], + "api_key": "fake-key", + **request_params, + } + return await litellm.acompletion(**request) if use_async else litellm.completion(**request) + + +class _PreCallParamsRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.litellm_params: tuple[object, ...] = () + + def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: + self.litellm_params = (*self.litellm_params, kwargs.get("litellm_params")) + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_malformed_routing_kwarg_answers_400_before_reaching_the_provider(use_async: bool): + with respx.mock(assert_all_called=False) as mock: + route: Final = mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer()) + with pytest.raises(litellm.BadRequestError, match="tpm") as raised: + await _openai_completion(use_async, tpm="abc") + + assert raised.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_malformed_routing_kwarg_is_dropped_from_the_whole_request_under_drop_params( + use_async: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _PreCallParamsRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock: + respx.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer()) + response: Final = await _openai_completion(use_async, tpm="abc", drop_params=True) + + assert response.choices[0].message.content == "hi" + assert len(recorder.litellm_params) == 1 + assert isinstance(recorder.litellm_params[0], Mapping) + assert recorder.litellm_params[0].get("tpm") is None From bd8a234beaab4b111674ffbfaf1b734a6bc0f969 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:22:21 +0000 Subject: [PATCH 08/12] fix(main): validate routing kwargs after the locals snapshot so fallbacks and model_list calls keep working The e2e test now uses gemini-3.8-flash with enough max_tokens to finish thinking before the tool call. --- litellm/main.py | 10 +++++----- .../test_gemini_tool_call_id_length_e2e.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 5a5c1adcc98..cc020d523b3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5480,11 +5480,6 @@ def completion( ### VALIDATE Request ### if model is None: raise ValueError("model param not passed in.") - supplemental_params: Final = _supplemental_provider_params( - _REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model - ) - for dropped_key in supplemental_params.dropped_keys: - kwargs.pop(dropped_key) # validate messages messages = validate_and_fix_openai_messages(messages=messages) tools = validate_and_fix_openai_tools(tools=tools) @@ -5496,6 +5491,11 @@ def completion( ######### unpacking kwargs ##################### args: Final = _locals_snapshot(locals()) + supplemental_params: Final = _supplemental_provider_params( + _REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model + ) + for dropped_key in supplemental_params.dropped_keys: + kwargs.pop(dropped_key) # Set by the responses->completion fallback so completion() does not bridge # back to the Responses API: that round-trip mutually recurses forever for a diff --git a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py index 883b6c08aa2..64fc3aa72af 100644 --- a/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py +++ b/tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py @@ -25,7 +25,7 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -BACKEND_MODEL: Final = "gemini/gemini-3.5-flash" +BACKEND_MODEL: Final = "gemini/gemini-3.8-flash" GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY" # bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20 AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") @@ -117,7 +117,7 @@ class TestGeminiToolCallIdLength: messages=[_user_message()], tools=[WEATHER_TOOL], tool_choice="required", - max_tokens=128, + max_tokens=2048, extra_body=NO_PROXY_CACHE, ) @@ -138,7 +138,7 @@ class TestGeminiToolCallIdLength: messages=[_user_message()], tools=[WEATHER_TOOL], tool_choice="required", - max_tokens=128, + max_tokens=2048, stream=True, extra_body=NO_PROXY_CACHE, ) @@ -164,7 +164,7 @@ class TestGeminiToolCallIdLength: messages=[question], tools=[WEATHER_TOOL], tool_choice="required", - max_tokens=128, + max_tokens=2048, extra_body=NO_PROXY_CACHE, ) calls: Final = _tool_calls(first) @@ -178,7 +178,7 @@ class TestGeminiToolCallIdLength: messages=[question, assistant, *(_tool_result(call) for call in calls)], tools=[WEATHER_TOOL], tool_choice="none", - max_tokens=128, + max_tokens=2048, extra_body=NO_PROXY_CACHE, ) From aa2195a995f67cffe63286aafe58c1bdefa0ee95 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:05:38 -0700 Subject: [PATCH 09/12] fix(gemini): keep non-chat routes tolerant of a malformed thought_signature_in_tool_call_id, match tool results by exact id first, and keep the deployment's value when drop_params drops a malformed override --- .../litellm_core_utils/get_litellm_params.py | 2 + .../prompt_templates/factory.py | 39 ++--- litellm/main.py | 13 +- litellm/router.py | 26 ++- litellm/types/router.py | 32 +++- .../router_code_coverage.py | 1 + .../test_thought_signature_in_tool_call_id.py | 158 ++++++++++++++++-- 7 files changed, 232 insertions(+), 39 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8ff374bb55f..121df45d8b6 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -96,6 +96,8 @@ OPTIONAL_KWARGS_KEYS: Final = ( # Backward-compatible alias for existing imports/tests. _OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS +SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"} + _CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions) _CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions)) _MAX_SHOWN_INT_BITS: Final = 64 diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 88bdba6d909..2f971fcb789 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1266,11 +1266,20 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str _encode_tool_call_id_with_signature = encode_tool_call_id_with_signature -def _same_tool_call(tool_call_id: str, other_tool_call_id: str) -> bool: - return ( - tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] - == other_tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] +def _tool_call_id_prefix(tool_call_id: object) -> str | None: + return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None + + +def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None: + exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id) + prefix: Final = _tool_call_id_prefix(tool_call_id) + prefix_matches: Final = tuple( + call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix ) + matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ()) + function: Final = matches[0].get("function") if matches else None + name: Final = function.get("name") if isinstance(function, Mapping) else None + return name if isinstance(name, str) else None def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: @@ -1712,22 +1721,14 @@ def convert_to_gemini_tool_call_result( ) ) is not None: media_parts.append(media_part) - name: str | None = message.get("name", "") - # Recover name from last message with tool calls - if last_message_with_tool_calls: - tools: Final = last_message_with_tool_calls.get("tool_calls", []) - msg_tool_call_id: Final = message.get("tool_call_id", None) - for tool in tools: - prev_tool_call_id = tool.get("id", None) - if ( - isinstance(msg_tool_call_id, str) - and isinstance(prev_tool_call_id, str) - and msg_tool_call_id - and prev_tool_call_id - and _same_tool_call(msg_tool_call_id, prev_tool_call_id) - ): - name = tool.get("function", {}).get("name", "") + msg_tool_call_id: Final = message.get("tool_call_id", None) + called_tool_name: Final = ( + _called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ()) + if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id + else None + ) + name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "") # Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix). gemini_call_id: str | None = None diff --git a/litellm/main.py b/litellm/main.py index 9321dd482e3..f1c6920fee3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -83,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( AWS_CREDENTIAL_KWARGS_KEYS, OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, - OPTIONAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY, + SUPPLEMENTAL_KWARGS_KEYS, InvalidControlOption, parse_control_options, with_control_options, @@ -154,7 +154,7 @@ from litellm.types.completion import ( ) from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues -from litellm.types.router import GenericLiteLLMParams +from litellm.types.router import STRICT_LITELLM_PARAMS, GenericLiteLLMParams from litellm.types.utils import ( BaseUrlRoutingHint, CustomPricingLiteLLMParams, @@ -398,13 +398,12 @@ class _SupplementalProviderParams: dropped_keys: frozenset[str] = frozenset() -_SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"} - - def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams: - supplemental: Final = {key: kwargs[key] for key in _SUPPLEMENTAL_KWARGS_KEYS if key in kwargs} + supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs} try: - return _SupplementalProviderParams(params=GenericLiteLLMParams.model_validate(supplemental)) + return _SupplementalProviderParams( + params=GenericLiteLLMParams.model_validate(supplemental, context=STRICT_LITELLM_PARAMS) + ) except ValidationError as error: invalid: Final = tuple(detail for detail in error.errors() if detail["loc"]) invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid) diff --git a/litellm/router.py b/litellm/router.py index f0bb6c3a7a9..61b0d2c5f00 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -96,6 +96,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, inferred_provider, @@ -387,6 +388,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.router import ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, + STRICT_LITELLM_PARAMS, VALID_LITELLM_ENVIRONMENTS, AlertingConfig, AllowedFailsPolicy, @@ -666,6 +668,24 @@ def _drops_malformed_request_tools( return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools") +def _malformed_request_params_the_deployment_sets( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> frozenset[str]: + overrides: Final = { + key: request[key] + for key in SUPPLEMENTAL_KWARGS_KEYS + if key in request and deployment_params.get(key) is not None + } + try: + GenericLiteLLMParams.model_validate(overrides, context=STRICT_LITELLM_PARAMS) + except ValidationError as error: + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + policy: Final = drop_policy({**deployment_params, **set_defaults, **request}) + malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"]) + return frozenset(key for key in malformed if policy.drops(key)) + return frozenset() + + def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None: if not isinstance(request_tools, (list, tuple)): return None @@ -4826,6 +4846,8 @@ class Router: if isinstance(raw_deployment_params, LiteLLM_Params) else raw_deployment_params ) + for key in _malformed_request_params_the_deployment_sets(deployment_context, defaults, request_context): + kwargs.pop(key) declared_provider: Final = deployment_context.get("custom_llm_provider") declared_model: Final = deployment_context.get("model") deployment_provider: Final = ( @@ -10462,7 +10484,9 @@ class Router: merged_params: Final[Mapping[str, object]] = ( _litellm_params if zeroed_pricing is None else MappingProxyType({**_litellm_params, **zeroed_pricing}) ) - litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(dict(**merged_params)) + litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate( + dict(**merged_params), context=STRICT_LITELLM_PARAMS + ) declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params) warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params) diff --git a/litellm/types/router.py b/litellm/types/router.py index ddf15c79c1b..691bde090f1 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,6 +6,7 @@ import datetime import enum from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -22,7 +23,19 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout -from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator +from pydantic import ( + BeforeValidator, + ConfigDict, + Field, + JsonValue, + StrictInt, + ValidationError, + ValidationInfo, + ValidatorFunctionWrapHandler, + WrapValidator, + field_validator, + model_validator, +) from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger @@ -517,6 +530,19 @@ def holds_secret_pointer(param_name: str) -> bool: _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) +STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True}) + + +def _none_when_malformed_outside_strict_validation( + value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo +) -> object: + try: + return handler(value) + except ValidationError: + if info.context is STRICT_LITELLM_PARAMS: + raise + return None + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ @@ -582,7 +608,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): ) model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) merge_reasoning_content_in_choices: bool | None = False - thought_signature_in_tool_call_id: bool | None = None + thought_signature_in_tool_call_id: Annotated[ + bool | None, WrapValidator(_none_when_malformed_outside_strict_validation) + ] = None model_info: dict | None = None mock_response: str | ModelResponse | Exception | object | None = None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index f4a295e697d..5f98d80ca63 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -122,6 +122,7 @@ ignored_function_names = [ "_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py "_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py + "_malformed_request_params_the_deployment_sets", # Run by _update_kwargs_with_deployment, tested through the drop_params deployment test in test_thought_signature_in_tool_call_id.py "_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py "_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 77682722dfa..4f3b6fa4793 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -11,7 +11,7 @@ enable_preview_features=True to be enabled. import json import re -from collections.abc import Mapping, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from itertools import chain from types import MappingProxyType from typing import Final @@ -28,9 +28,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( get_thought_signature_from_tool, convert_to_gemini_tool_call_invoke, ) +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.llms.vertex_ai import HttpxPartType @@ -300,13 +302,11 @@ def test_parallel_tool_calls_with_signatures(enable_preview_features): GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*" BOUNDED_TOOL_CALL_ID: Final = re.compile(r"[A-Za-z0-9_-]{1,64}") LONG_SIGNATURE: Final = "Ab+/" * 200 -WEATHER_TOOL: Final = { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - }, +WEATHER_FUNCTION: Final = { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, } +WEATHER_TOOL: Final = {"type": "function", "function": WEATHER_FUNCTION} USER_TURN: Final = {"role": "user", "content": "What is the weather in Paris?"} @@ -385,9 +385,10 @@ def _returned_tool_calls( api_key="test-gemini-key", **request_params, ) + assert isinstance(response, CustomStreamWrapper) == stream returned_calls: Final = ( chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in response) - if stream + if isinstance(response, CustomStreamWrapper) else response.choices[0].message.tool_calls ) tool_calls: Final = [call.model_dump() for call in returned_calls] @@ -512,6 +513,7 @@ async def _messages_tool_use_blocks(stream: bool) -> list[Mapping[str, object]]: ) if not stream: return [block for block in response["content"] if block["type"] == "tool_use"] + assert isinstance(response, AsyncIterator) relayed: Final = b"".join([chunk async for chunk in response]) events: Final = [json.loads(line.removeprefix(b"data: ")) for line in relayed.splitlines() if line.startswith(b"data:")] return [event["content_block"] for event in events if _is_tool_use_start(event)] @@ -529,7 +531,7 @@ async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id( assert tool_uses[0]["provider_specific_fields"]["signature"] == LONG_SIGNATURE -RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_TOOL["function"]} +RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_FUNCTION} SHORT_IDS: Final = MappingProxyType({"thought_signature_in_tool_call_id": False}) @@ -544,9 +546,10 @@ def _responses_function_call_items(stream: bool, request_params: Mapping[str, ob api_key="test-gemini-key", **request_params, ) + assert isinstance(response, Iterator) == stream items: Final = ( [event.item for event in response if event.type == "response.output_item.done"] - if stream + if isinstance(response, Iterator) and not isinstance(response, ResponsesAPIResponse) else response.output ) return [item for item in items if getattr(item, "type", None) == "function_call"] @@ -724,3 +727,138 @@ def test_deployment_setting_rejects_a_value_that_is_not_a_boolean(): } ] ) + + +def test_deployment_setting_survives_a_request_value_that_drop_params_drops(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gemini-agentcore", + "litellm_params": { + "model": "gemini/gemini-3.5-flash", + "api_key": "test-gemini-key", + "thought_signature_in_tool_call_id": False, + }, + } + ] + ) + with respx.mock: + respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) + response: Final = router.completion( + model="gemini-agentcore", + messages=[USER_TURN], + tools=[WEATHER_TOOL], + thought_signature_in_tool_call_id="maybe", + drop_params=True, + ) + + assert _is_bounded(response.choices[0].message.tool_calls[0].id) + + +def test_responses_request_setting_that_is_not_a_boolean_answers_400(): + with pytest.raises(litellm.BadRequestError, match="thought_signature_in_tool_call_id") as raised: + _responses_function_call_items(False, {"thought_signature_in_tool_call_id": "maybe"}) + + assert raised.value.status_code == 400 + + +def test_responses_request_setting_that_is_not_a_boolean_is_dropped_under_drop_params(): + function_calls: Final = _responses_function_call_items( + False, {"thought_signature_in_tool_call_id": "maybe", "drop_params": True} + ) + + assert len(function_calls) == 1 + assert get_thought_signature_from_tool({"id": function_calls[0].call_id}) == LONG_SIGNATURE + + +@pytest.mark.parametrize("drop_params", [False, True]) +async def test_messages_request_setting_that_is_not_a_boolean_follows_drop_params( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=False) as mocked: + mocked.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, False)) + request: Final = litellm.anthropic.messages.acreate( + max_tokens=256, + messages=[USER_TURN], + model="gemini/gemini-3.5-flash", + tools=[ANTHROPIC_WEATHER_TOOL], + api_key="test-gemini-key", + thought_signature_in_tool_call_id="maybe", + drop_params=drop_params, + ) + if not drop_params: + with pytest.raises(litellm.BadRequestError, match="thought_signature_in_tool_call_id"): + await request + return + response: Final = await request + + tool_uses: Final = [block for block in response["content"] if block["type"] == "tool_use"] + assert len(tool_uses) == 1 + + +def test_setting_that_is_not_a_boolean_is_ignored_where_nothing_reads_it(): + embedding_vector: Final = [0.25, 0.5] + with respx.mock: + route: Final = respx.post("https://api.openai.com/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": embedding_vector}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + response: Final = litellm.embedding( + model="openai/text-embedding-3-small", + input=["hi"], + api_key="test-openai-key", + thought_signature_in_tool_call_id="maybe", + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + assert response.data[0]["embedding"] == embedding_vector + assert "thought_signature_in_tool_call_id" not in sent_body + + +TIME_TOOL: Final = { + "type": "function", + "function": {"name": "get_time", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, +} + + +@pytest.mark.parametrize( + "tool_call_ids", + [ + ("call_1__thought__S1", "call_1__thought__S2"), + ("__thought__S1", "__thought__S2"), + ], +) +def test_tool_results_keep_their_own_names_when_tool_call_ids_share_a_prefix(tool_call_ids: tuple[str, str]): + called_tools: Final = (("get_weather", tool_call_ids[0]), ("get_time", tool_call_ids[1])) + with respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) + litellm.completion( + model="gemini/gemini-3.5-flash", + messages=[ + USER_TURN, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": name, "arguments": '{"city": "Paris"}'}} + for name, call_id in called_tools + ], + }, + *({"role": "tool", "tool_call_id": call_id, "content": f'{{"{name}": 1}}'} for name, call_id in called_tools), + ], + tools=[WEATHER_TOOL, TIME_TOOL], + api_key="test-gemini-key", + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + parts: Final = chain.from_iterable(turn["parts"] for turn in sent_body["contents"]) + sent_names: Final = [part["function_response"]["name"] for part in parts if "function_response" in part] + assert sent_names == ["get_weather", "get_time"] From bce6d2893f88eb6d3846ada21ff73dd840ea5ca8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:47:31 -0700 Subject: [PATCH 10/12] fix(router): validate strict litellm params through a helper so star imports cannot re-bind a public Final --- litellm/main.py | 6 ++---- litellm/router.py | 8 ++++---- litellm/types/router.py | 13 +++++++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index ec024eee433..44a595cbc43 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -154,7 +154,7 @@ from litellm.types.completion import ( ) from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues -from litellm.types.router import STRICT_LITELLM_PARAMS, GenericLiteLLMParams +from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly from litellm.types.utils import ( BaseUrlRoutingHint, CustomPricingLiteLLMParams, @@ -401,9 +401,7 @@ class _SupplementalProviderParams: def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams: supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs} try: - return _SupplementalProviderParams( - params=GenericLiteLLMParams.model_validate(supplemental, context=STRICT_LITELLM_PARAMS) - ) + return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental)) except ValidationError as error: invalid: Final = tuple(detail for detail in error.errors() if detail["loc"]) invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid) diff --git a/litellm/router.py b/litellm/router.py index 2a614eaf086..3a5a4c1a6d9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -414,7 +414,6 @@ from litellm.types.llms.openai import ( ) from litellm.types.router import ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, - STRICT_LITELLM_PARAMS, VALID_LITELLM_ENVIRONMENTS, AlertingConfig, AllowedFailsPolicy, @@ -451,6 +450,7 @@ from litellm.types.router import ( SearchToolTypedDict, TaggedPreRoutingStrategy, holds_secret_pointer, + validate_litellm_params_strictly, ) from litellm.types.services import ServiceTypes from litellm.types.utils import ( @@ -704,7 +704,7 @@ def _malformed_request_params_the_deployment_sets( if key in request and deployment_params.get(key) is not None } try: - GenericLiteLLMParams.model_validate(overrides, context=STRICT_LITELLM_PARAMS) + validate_litellm_params_strictly(GenericLiteLLMParams, overrides) except ValidationError as error: set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} policy: Final = drop_policy({**deployment_params, **set_defaults, **request}) @@ -10628,8 +10628,8 @@ class Router: merged_params: Final[Mapping[str, object]] = ( _litellm_params if zeroed_pricing is None else MappingProxyType({**_litellm_params, **zeroed_pricing}) ) - litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate( - dict(**merged_params), context=STRICT_LITELLM_PARAMS + litellm_params: Final[LiteLLM_Params] = validate_litellm_params_strictly( + LiteLLM_Params, dict(**merged_params) ) declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params) warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params) diff --git a/litellm/types/router.py b/litellm/types/router.py index 3a559f13cde..87a293fc4d2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -24,6 +24,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout from pydantic import ( + BaseModel, BeforeValidator, ConfigDict, Field, @@ -530,7 +531,15 @@ def holds_secret_pointer(param_name: str) -> bool: _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) -STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True}) +_STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True}) + +_StrictlyValidatedParams: Final = TypeVar("_StrictlyValidatedParams", bound=BaseModel) + + +def validate_litellm_params_strictly( + params_model: type[_StrictlyValidatedParams], params: Mapping[str, object] +) -> _StrictlyValidatedParams: + return params_model.model_validate(params, context=_STRICT_LITELLM_PARAMS) def _none_when_malformed_outside_strict_validation( @@ -539,7 +548,7 @@ def _none_when_malformed_outside_strict_validation( try: return handler(value) except ValidationError: - if info.context is STRICT_LITELLM_PARAMS: + if info.context is _STRICT_LITELLM_PARAMS: raise return None From 33afbe19d5b6ef6139c700e2990b8c2632abf511 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:30:36 -0700 Subject: [PATCH 11/12] perf(router): skip strict validation when a request overrides no deployment param --- litellm/router.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 3a5a4c1a6d9..70a0036f97c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -699,10 +699,12 @@ def _malformed_request_params_the_deployment_sets( deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] ) -> frozenset[str]: overrides: Final = { - key: request[key] - for key in SUPPLEMENTAL_KWARGS_KEYS - if key in request and deployment_params.get(key) is not None + key: value + for key, value in request.items() + if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None } + if not overrides: + return frozenset() try: validate_litellm_params_strictly(GenericLiteLLMParams, overrides) except ValidationError as error: From 765b129ab49badf60f08cab8a19ffcbb0f389ed7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:23:59 -0700 Subject: [PATCH 12/12] fix(responses): keep a dict tool call's thought signature on the streamed function_call done item --- .../streaming_iterator.py | 7 +++++- .../test_litellm_completion_responses.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 0721968441c..394f221e0d0 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -93,7 +93,12 @@ def _tool_call_objects(tool_calls: object) -> tuple[object, ...]: def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]: - fields: Final[object] = getattr(tool_call, "provider_specific_fields", None) + mapping: Final = as_str_mapping(tool_call) + fields: Final[object] = ( + mapping.get("provider_specific_fields") + if mapping is not None + else getattr(tool_call, "provider_specific_fields", None) + ) return {"provider_specific_fields": fields} if fields else {} diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 7a5254b64f4..a6a1335fdc4 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4746,6 +4746,29 @@ class TestEnsureOutputItemContentPartAdded: assert done.item.name == "spawn_agent" assert done.item.namespace == "collaboration" + def test_streaming_final_only_dict_tool_call_keeps_its_provider_specific_fields(self): + from unittest.mock import MagicMock + + iterator = self._make_iterator() + iterator.tool_call_provider_fields = True + provider_specific_fields = {"thought_signature": "sig-" + "x" * 512} + message = MagicMock() + message.tool_calls = [ + { + "id": "call_1", + "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'}, + "provider_specific_fields": provider_specific_fields, + } + ] + complete_response = MagicMock() + complete_response.choices = [MagicMock(message=message)] + + iterator._queue_final_tool_call_done_events(complete_response) + + done = iterator._pending_tool_events[-1] + assert done.type == "response.output_item.done" + assert done.item.provider_specific_fields == provider_specific_fields + def test_streaming_top_level_function_collision_stays_unnamespaced(self): iterator = self._make_iterator() iterator.responses_api_request = {