diff --git a/litellm/main.py b/litellm/main.py index 701a16fda32..e3b4e5800e4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy +from dataclasses import dataclass from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -381,18 +382,27 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return assert_never(control) -def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> GenericLiteLLMParams | None: +@dataclass(frozen=True, slots=True) +class _SupplementalProviderParams: + params: GenericLiteLLMParams | None + dropped_keys: frozenset[str] = frozenset() + + +def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams: supplemental: Final = {key: kwargs[key] for key in OPTIONAL_KWARGS_KEYS if key in kwargs} if not supplemental: - return None + return _SupplementalProviderParams(params=None) try: - return GenericLiteLLMParams.model_validate(supplemental) + return _SupplementalProviderParams(params=GenericLiteLLMParams.model_validate(supplemental)) except ValidationError as error: invalid: Final = tuple(detail for detail in error.errors() if detail["loc"]) invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid) if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: - return GenericLiteLLMParams.model_validate( - {key: value for key, value in supplemental.items() if key not in invalid_keys} + return _SupplementalProviderParams( + params=GenericLiteLLMParams.model_validate( + {key: value for key, value in supplemental.items() if key not in invalid_keys} + ), + dropped_keys=invalid_keys, ) raise litellm.BadRequestError( message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid), @@ -701,7 +711,7 @@ async def acompletion( "str | None", request_kwargs.get("api_base") ) or base_url, - litellm_params=_supplemental_provider_params(request_kwargs, model), + litellm_params=_supplemental_provider_params(request_kwargs, model).params, ) fallbacks = fallbacks or litellm.model_fallbacks @@ -5446,6 +5456,9 @@ def completion( ### VALIDATE Request ### if model is None: raise ValueError("model param not passed in.") + supplemental_params: Final = _supplemental_provider_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), model) + for dropped_key in supplemental_params.dropped_keys: + kwargs.pop(dropped_key) # validate messages messages = validate_and_fix_openai_messages(messages=messages) tools = validate_and_fix_openai_tools(tools=tools) @@ -5682,10 +5695,7 @@ def completion( custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=_supplemental_provider_params( - routing_kwargs, - cast(str, model), # cast-ok: model is the alias-resolved request model name - ), + litellm_params=supplemental_params.params, ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 5489d2223ac..8a6684917a3 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -8,6 +8,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.responses.additional_tools import hoist_additional_tools from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -99,6 +100,9 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, litellm_metadata=kwargs.get("litellm_metadata", {}), + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id( + _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") @@ -150,5 +154,8 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=responses_api_request, custom_llm_provider=litellm_completion_request.get("custom_llm_provider"), litellm_metadata=kwargs.get("litellm_metadata", {}), + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id( + _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 35160830fdc..34903eda221 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -85,6 +85,7 @@ class ResponsesSessionHandler: previous_response_id: str, item_cache_control: bool = False, client_response_ids: tuple[str, ...] = (), + tool_call_provider_fields: bool = False, ) -> ChatCompletionSession: """ Return the chat completion message history for a previous response id @@ -125,6 +126,7 @@ class ResponsesSessionHandler: proxy_server_request_dict=proxy_server_request_dict, chat_completion_message_history=chat_completion_message_history, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) @@ -150,6 +152,7 @@ class ResponsesSessionHandler: | Message ], item_cache_control: bool = False, + tool_call_provider_fields: bool = False, ): """ Extend the chat completion message history with the spend log payload @@ -182,6 +185,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) @@ -196,6 +200,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + tool_call_provider_fields=tool_call_provider_fields, ) ) elif isinstance(_response_output, dict) and _response_output and _response_output != {}: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index cb074c86b38..4b41797bb46 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -127,6 +127,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None = None, litellm_metadata: dict | None = None, + tool_call_provider_fields: bool = False, ): self.model: str = model self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper @@ -134,6 +135,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request self.custom_llm_provider: str | None = custom_llm_provider self.litellm_metadata: dict | None = litellm_metadata or {} + self.tool_call_provider_fields: Final = tool_call_provider_fields # Store lightweight dict snapshots for stream_chunk_builder to reduce # repeated Pydantic attribute access in end-of-stream assembly. self.collected_chat_completion_chunks: list[dict[str, object]] = [] @@ -488,7 +490,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=output_index, sequence_number=self._sequence_number, - item=BaseLiteLLMOpenAIResponseObject(**item_kwargs, **_tool_call_provider_specific_fields(tc)), + item=BaseLiteLLMOpenAIResponseObject( + **item_kwargs, + **(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}), + ), ) self._pending_tool_events.append(item_done_event) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 1c37aaac6fe..78c0fae3fe2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -55,6 +55,7 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support @@ -327,17 +328,32 @@ def _dropped_cache_control(_: object) -> None: return None -def _without_cache_control_key(item: object) -> object: +def _without_keys(item: object, keys: frozenset[str]) -> object: if not isinstance(item, dict): return item - return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"} + return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys} + + +def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam: + if not keys or not isinstance(input, list): + return input + return cast( # cast-ok: the same input items, each minus the given keys + ResponseInputParam, [_without_keys(item, keys) for item in input] + ) def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam: - if not isinstance(input, list): - return input - return cast( # cast-ok: the same input items, each minus its own cache_control key - ResponseInputParam, [_without_cache_control_key(item) for item in input] + return _without_item_keys(input, frozenset(("cache_control",))) + + +def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]: + return frozenset( + key + for key, kept in ( + ("cache_control", item_cache_control), + ("provider_specific_fields", tool_call_provider_fields), + ) + if not kept ) @@ -879,6 +895,7 @@ class LiteLLMCompletionResponsesConfig: responses_api_request=responses_api_request, replay_reasoning=True, item_cache_control=keeps_item_marks, + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), ) litellm_completion_request: dict = { @@ -940,6 +957,7 @@ class LiteLLMCompletionResponsesConfig: responses_api_request: ResponsesAPIOptionalRequestParams | dict, replay_reasoning: bool = False, item_cache_control: bool = False, + tool_call_provider_fields: bool = False, ) -> list[ AllMessageValues | GenericChatCompletionMessage @@ -955,7 +973,9 @@ class LiteLLMCompletionResponsesConfig: assistant ``reasoning_content`` and signed ``thinking_blocks`` so the provider gets its own chain-of-thought back instead of reading it as visible text. ``item_cache_control`` belongs to the same callers, when - the target keeps an input item's ``cache_control`` on the message. + the target keeps an input item's ``cache_control`` on the message, and + ``tool_call_provider_fields`` when the target reads a function call's + thought signature from ``provider_specific_fields`` instead of its id. Callers that only inspect the messages (token counting, rate limiting, guardrail scanning) leave both off, because they need every piece of text @@ -978,7 +998,7 @@ class LiteLLMCompletionResponsesConfig: messages.extend( LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=input if item_cache_control else without_item_cache_control(input), + input=_without_item_keys(input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)), replay_reasoning=replay_reasoning, ) ) @@ -1021,6 +1041,7 @@ class LiteLLMCompletionResponsesConfig: ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, item_cache_control=keeps_item_marks, client_response_ids=previous_response_client_ids, + tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), ) chat_completion_session: Final = ( await lookup(previous_response_id) diff --git a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 6c3aed5b04f..77682722dfa 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -13,6 +13,7 @@ import json import re from collections.abc import Mapping, Sequence from itertools import chain +from types import MappingProxyType from typing import Final import httpx @@ -529,9 +530,10 @@ async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id( RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_TOOL["function"]} +SHORT_IDS: Final = MappingProxyType({"thought_signature_in_tool_call_id": False}) -def _responses_function_call_items(stream: bool) -> list[object]: +def _responses_function_call_items(stream: bool, request_params: Mapping[str, object] = SHORT_IDS) -> list[object]: with respx.mock: respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, stream)) response: Final = litellm.responses( @@ -540,7 +542,7 @@ def _responses_function_call_items(stream: bool) -> list[object]: tools=[RESPONSES_WEATHER_TOOL], stream=stream, api_key="test-gemini-key", - thought_signature_in_tool_call_id=False, + **request_params, ) items: Final = ( [event.item for event in response if event.type == "response.output_item.done"] @@ -559,6 +561,60 @@ def test_responses_function_call_keeps_thought_signature_kept_out_of_its_id(stre assert function_calls[0].provider_specific_fields["thought_signature"] == LONG_SIGNATURE +def test_streamed_function_call_carries_its_thought_signature_only_in_its_id_by_default(): + function_calls: Final = _responses_function_call_items(True, {}) + + assert len(function_calls) == 1 + assert get_thought_signature_from_tool({"id": function_calls[0].call_id}) == LONG_SIGNATURE + assert getattr(function_calls[0], "provider_specific_fields", None) is None + + +def _deepseek_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "deepseek-v4-flash", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sunny"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + + +@pytest.mark.parametrize("request_params", [{}, SHORT_IDS]) +def test_responses_function_call_provider_fields_reach_the_provider_only_with_short_ids( + request_params: Mapping[str, object], +): + replayed_item: Final = { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "provider_specific_fields": {"thought_signature": LONG_SIGNATURE}, + } + + with respx.mock: + route: Final = respx.post(url__regex=r"https://api\.deepseek\.com/.*").mock(return_value=_deepseek_answer()) + litellm.responses( + model="deepseek/deepseek-v4-flash", + input=[ + {"role": "user", "content": USER_TURN["content"]}, + replayed_item, + {"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'}, + ], + tools=[RESPONSES_WEATHER_TOOL], + api_key="test-deepseek-key", + **request_params, + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + sent_tool_calls: Final = list(chain.from_iterable(message.get("tool_calls") or () for message in sent_body["messages"])) + assert len(sent_tool_calls) == 1 + assert ("provider_specific_fields" in sent_tool_calls[0]) is bool(request_params) + + @pytest.mark.parametrize("client_keeps_provider_fields", [True, False]) def test_responses_function_call_replays_its_thought_signature_to_gemini(client_keeps_provider_fields: bool): function_call: Final = _responses_function_call_items(False)[0] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f34ca7d0ff8..674c65fb130 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4318,6 +4318,7 @@ class TestEnsureOutputItemContentPartAdded: iterator._queued_web_search_call_ids = set() iterator._tool_search_requested = False iterator._tool_search_call_ids = set() + iterator.tool_call_provider_fields = False return iterator def _make_text_chunk(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 7a146c1b665..26823ba9534 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1,11 +1,14 @@ import json import sqlite3 from collections.abc import Sequence +from itertools import chain from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, patch import pytest +import httpx +import respx from fastapi import HTTPException from fastapi.testclient import TestClient @@ -955,3 +958,69 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_ assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"] assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"] + + +GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*" +LONG_SIGNATURE: Final = "Ab+/" * 200 + + +def _gemini_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10}, + }, + ) + + +def _stored_function_call_turn(response_id: str) -> dict: + return { + "request_id": response_id, + "call_type": "aresponses", + "session_id": "gemini-session", + "proxy_server_request": {"input": "What is the weather in Paris?", "model": "gemini-short-ids"}, + "response": { + "id": response_id, + "object": "response", + "output": [ + { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + "provider_specific_fields": {"thought_signature": LONG_SIGNATURE}, + } + ], + }, + } + + +@pytest.mark.parametrize("short_ids", [False, True]) +async def test_previous_response_replays_a_thought_signature_kept_out_of_the_id_only_with_short_ids( + short_ids: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="gemini", model_id=None, response_id="chatcmpl-gemini-turn-1" + ) + stored: Final = _StoredSpendLogs((_stored_function_call_turn(response_id),)) + + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored)), respx.mock: + route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer()) + await litellm.aresponses( + model="gemini/gemini-3.5-flash", + input=[{"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'}], + previous_response_id=response_id, + api_key="test-gemini-key", + **({"thought_signature_in_tool_call_id": False} if short_ids else {}), + ) + sent_body: Final = json.loads(route.calls.last.request.content) + + model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"] + model_parts: Final = list(chain.from_iterable(turn["parts"] for turn in model_turns)) + assert len(model_parts) == 1 + assert (model_parts[0]["thoughtSignature"] == LONG_SIGNATURE) is short_ids diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f5d13aecb6f..d6f97692eec 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -9810,3 +9810,64 @@ async def test_converse_fallback_prices_the_current_attempt( else: client.close() configuration.reset_rust_configuration() + + +def _openai_chat_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +async def _openai_completion(use_async: bool, **request_params: object) -> litellm.ModelResponse: + request: Final = { + "model": "openai/gpt-5.6", + "messages": [{"role": "user", "content": "hi"}], + "api_key": "fake-key", + **request_params, + } + return await litellm.acompletion(**request) if use_async else litellm.completion(**request) + + +class _PreCallParamsRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.litellm_params: tuple[object, ...] = () + + def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: + self.litellm_params = (*self.litellm_params, kwargs.get("litellm_params")) + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_malformed_routing_kwarg_answers_400_before_reaching_the_provider(use_async: bool): + with respx.mock(assert_all_called=False) as mock: + route: Final = mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer()) + with pytest.raises(litellm.BadRequestError, match="tpm") as raised: + await _openai_completion(use_async, tpm="abc") + + assert raised.value.status_code == 400 + assert not route.called + + +@pytest.mark.parametrize("use_async", [False, True]) +async def test_malformed_routing_kwarg_is_dropped_from_the_whole_request_under_drop_params( + use_async: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _PreCallParamsRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock: + respx.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer()) + response: Final = await _openai_completion(use_async, tpm="abc", drop_params=True) + + assert response.choices[0].message.content == "hi" + assert len(recorder.litellm_params) == 1 + assert isinstance(recorder.litellm_params[0], Mapping) + assert recorder.litellm_params[0].get("tpm") is None