From aa82b4c652d273666ac81b518461623ba7d16cd7 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Thu, 5 Nov 2026 13:02:54 -0800 Subject: [PATCH 01/18] fix(responses): keep the system prompt in input for json_object on the chat to responses bridge (#49957) Backport of #49957 to lts/1.108.x. Cherry-picked from 9703e01cd4704b3b6a850efaee3f15d314403ce8 (main). --- .../transformation.py | 18 ++++---- ...responses_transformation_transformation.py | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8d23526b41a..822ef1ea0be 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1066,11 +1066,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, ) - # OpenAI's Responses API rejects an empty input. For a system-only - # request, carry the system message as a system-role input item instead - # of instructions, mirroring how non-string system content is already - # handled in convert_chat_completion_messages_to_responses_api. - is_system_only_request: Final = not converted_input_items and converted_instructions is not None + # Keep the system text in input instead of instructions when the Responses API needs it there. + # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions. + typed_optional_params: Final = cast("Mapping[str, object]", optional_params) # cast-ok: untyped dict + system_goes_in_input: Final = converted_instructions is not None and ( + not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object" + ) target_input_items: Final = ( _without_audio_input_parts(converted_input_items) if _drops_audio_input(model, litellm_params) @@ -1082,12 +1083,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "type": "message", "role": "system", "content": [{"type": "input_text", "text": converted_instructions}], - } + }, + *target_input_items, ] - if is_system_only_request + if system_goes_in_input else target_input_items ) - instructions: Final = None if is_system_only_request else converted_instructions + instructions: Final = None if system_goes_in_input else converted_instructions # Build responses API request using the reverse transformation logic responses_api_request: Final = ResponsesAPIOptionalRequestParams() diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 83c59480d37..eb308a2c513 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1291,6 +1291,51 @@ def test_transform_request_system_only_message_maps_to_system_input_item(): assert not result.get("instructions") +_JSON_SYSTEM_PROMPT: Final = "You output JSON objects." +_PLAIN_USER_ITEM: Final = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "List three colors."}], +} +_COLORS_SCHEMA: Final = { + "type": "json_schema", + "json_schema": {"name": "colors", "schema": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize( + ("response_format", "expected_input", "expected_instructions"), + [ + ( + {"type": "json_object"}, + [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": _JSON_SYSTEM_PROMPT}]}, + _PLAIN_USER_ITEM, + ], + None, + ), + (_COLORS_SCHEMA, [_PLAIN_USER_ITEM], _JSON_SYSTEM_PROMPT), + ], + ids=["json_object_keeps_system_in_input", "json_schema_keeps_system_in_instructions"], +) +def test_transform_request_json_object_keeps_system_prompt_where_the_json_word_check_reads_it( + response_format, expected_input, expected_instructions +): + result = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-5.4-mini", + messages=[ + {"role": "system", "content": _JSON_SYSTEM_PROMPT}, + {"role": "user", "content": "List three colors."}, + ], + optional_params={"response_format": response_format}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert (result["input"], result.get("instructions")) == (expected_input, expected_instructions), result + + def test_transform_request_single_char_keys_not_matched(): """Test that single-character keys are not incorrectly matched to 'metadata' or 'previous_response_id' From d1d159637e3551bd82e36858ae3d2cecee52a1fa Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Thu, 5 Nov 2026 13:37:14 -0800 Subject: [PATCH 02/18] fix(router): preserve tag fail-open policy through affinity and cooldown (#48470) Adapted for lts/1.108.x: the line has no _async_routable_cooled_deployment_ids, so its call site and the _pool test helper change are dropped, and the test file gains the RouterErrors import it lacks on this line. (cherry picked from commit cfe0100e1fca613e7b1af27dd1fe4807ca4d1d1a) --- litellm/router.py | 1 + litellm/router_strategy/tag_based_routing.py | 23 ++- .../test_router_tag_regex_routing.py | 11 ++ .../test_router_tag_routing.py | 166 +++++++++++++++++- 4 files changed, 189 insertions(+), 12 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 41706bdf539..244fac94d42 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14072,6 +14072,7 @@ class Router: request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + request_deployments=request_deployments, ) # narrow to whatever `self.routing_plugins` left in candidate_models diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 541fdd19cff..4ca4c17f6f9 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -8,9 +8,11 @@ Use this to route requests between Teams """ import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload + +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY @@ -50,6 +52,10 @@ class _DeploymentLike(Protocol): def get(self, key: Literal["model_name"], /) -> object: ... +class _DeploymentWithModelInfo(TypedDict, total=False): + model_info: ReadOnly[Mapping[str, object]] + + class _MetadataLike(Protocol): @overload def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ... @@ -279,13 +285,14 @@ def _unknown_required_tag_hides_an_answer( def _chain_allows_fail_open( healthy_deployments: _DeploymentPool, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], excluded_set: frozenset[str], required_set: frozenset[str], routing_confirmed: frozenset[str], ) -> bool: if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed): return False - return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments) + return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments) def _trusted_only_pool( @@ -321,6 +328,7 @@ def _trusted_only_pool( def _resolve_or_fail_open( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], pool: Sequence[_DeploymentLike], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], @@ -333,7 +341,7 @@ def _resolve_or_fail_open( ) -> tuple[_DeploymentLike, ...]: if pool: return tuple(pool) - if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed): + if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed): # Fall open only within whatever still satisfies whichever constraints # trace back to key/team policy. A constraint with no inherited backing at # all (or, when inherited_tags is unavailable, any constraint at all) can @@ -350,6 +358,7 @@ def _resolve_or_fail_open( def _resolve_constraint_only_pool( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], required_set: frozenset[str], @@ -365,6 +374,7 @@ def _resolve_constraint_only_pool( else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set) ) return _resolve_or_fail_open( + request_deployments, pool, healthy_deployments, excluded_set, @@ -480,6 +490,8 @@ async def get_deployments_for_tag( healthy_deployments: _DeploymentPool, request_kwargs: _RequestKwargsLike | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + *, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], ) -> _DeploymentPool: """ Returns a list of deployments that match the requested model and tags in the request. @@ -548,6 +560,7 @@ async def get_deployments_for_tag( if constraint_only: return _resolve_constraint_only_pool( + request_deployments, healthy_deployments, excluded_set, required_set, @@ -599,6 +612,7 @@ async def get_deployments_for_tag( if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, @@ -616,6 +630,7 @@ async def get_deployments_for_tag( and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed) ): return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, diff --git a/tests/unit/router_strategy/test_router_tag_regex_routing.py b/tests/unit/router_strategy/test_router_tag_regex_routing.py index 212c2627d88..46e79a1b8c8 100644 --- a/tests/unit/router_strategy/test_router_tag_regex_routing.py +++ b/tests/unit/router_strategy/test_router_tag_regex_routing.py @@ -121,6 +121,7 @@ async def test_claude_code_ua_routes_to_cc_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.2.3"}}, ) assert len(result) == 1 @@ -135,6 +136,7 @@ async def test_regular_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (browser)"}}, ) assert len(result) == 1 @@ -149,6 +151,7 @@ async def test_no_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {}}, ) assert len(result) == 1 @@ -164,6 +167,7 @@ async def test_tag_routing_metadata_written_for_regex_match(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": metadata}, ) assert "tag_routing" in metadata @@ -181,6 +185,7 @@ async def test_tag_filtering_disabled_returns_all_deployments(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert result == ALL_DEPLOYMENTS @@ -199,6 +204,7 @@ async def test_empty_healthy_deployments_with_request_tags_returns_empty_list(): llm_router_instance=router, model="gpt-5.2", healthy_deployments=[], + request_deployments=[], request_kwargs={ "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]} }, @@ -228,6 +234,7 @@ async def test_explicit_tag_match_takes_precedence_over_regex(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_with_both], + request_deployments=[deployment_with_both], request_kwargs={"metadata": metadata}, ) assert len(result) == 1 @@ -269,6 +276,7 @@ async def test_user_agent_present_no_tag_regex_deployments_does_not_raise(): llm_router_instance=router, model="gpt-4", healthy_deployments=plain_tag_only_deployments, + request_deployments=plain_tag_only_deployments, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (any-client)"}}, ) # Falls through to "return healthy_deployments" path unchanged @@ -305,6 +313,7 @@ async def test_tag_routing_metadata_not_overwritten_for_multiple_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_a, deployment_b], + request_deployments=[deployment_a, deployment_b], request_kwargs={"metadata": metadata}, ) assert len(result) == 2 @@ -357,6 +366,7 @@ async def test_match_any_false_strict_tag_check_blocks_regex_fallback(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_strict, default_deployment], + request_deployments=[deployment_strict, default_deployment], request_kwargs={"metadata": metadata}, ) ids = [d["model_info"]["id"] for d in result] @@ -387,6 +397,7 @@ async def test_match_any_false_regex_only_deployment_still_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[regex_only_deployment], + request_deployments=[regex_only_deployment], request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert len(result) == 1 diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 57c0177f9b4..5738bf86ccc 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3,12 +3,15 @@ import logging -from typing import Final +from typing import Final, Literal +from unittest.mock import patch import pytest import litellm from litellm._logging import verbose_logger +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.router import RouterErrors async def _routed_model_ids( @@ -1365,14 +1368,17 @@ def test_chain_allows_fail_open_true_when_any_member_sets_flag(): {"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}}, {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) + is True + ) def test_chain_allows_fail_open_false_by_default(): from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open deployments = [{"model_info": {}}, {}] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset(), frozenset()) is False def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): @@ -1384,7 +1390,10 @@ def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): deployments = [ {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) + is True + ) def test_unknown_required_tag_hides_an_answer_denies_fail_open(): @@ -1405,7 +1414,11 @@ def test_unknown_required_tag_hides_an_answer_denies_fail_open(): # fail-open must be denied even though the flag is set on the group. assert ( _chain_allows_fail_open( - deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset() + deployments, + deployments, + frozenset(), + frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), + frozenset(), ) is False ) @@ -1432,7 +1445,9 @@ def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden(): # carries both, so this is a genuinely unsatisfiable combination, not an invented # tag masking a narrower answer. Fail-open must proceed normally. assert ( - _chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()) + _chain_allows_fail_open( + deployments, deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset() + ) is True ) @@ -1494,8 +1509,8 @@ def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confi }, ] required_set = frozenset({"provider:anthropic", "typo-tag"}) - assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False - assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, required_set) is True # --- get_deployments_for_tag required-AND ("&") integration tests --- @@ -1763,6 +1778,141 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default() # --- get_deployments_for_tag allow_fail_open integration tests --- +def _fail_open_router(opted_in_group: str | None = "chat", route: str = "model") -> litellm.Router: + model_name: Final = "chat-*" if route == "wildcard" else "chat" + team_info: Final = {"team_id": "team-a", "team_public_model_name": "public-chat"} if route == "team" else {} + return litellm.Router( + model_list=[ + { + "model_name": model_name if opted_in_group in (None, "chat") else opted_in_group, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["vision"]}, + "model_info": {"id": "opted-in", "allow_fail_open": opted_in_group is not None, **team_info}, + }, + { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["region:eu"]}, + "model_info": {"id": "remaining", **team_info}, + }, + ], + routing_groups=( + [{"group_name": "public-chat", "models": ["chat"], "routing_strategy": "simple-shuffle"}] + if route == "routing_group" + else None + ), + model_group_alias={"public-chat": "chat"} if route == "alias" else {}, + enable_tag_filtering=True, + optional_pre_call_checks=["responses_api_deployment_check"], + ) + + +def _response_id_for_remaining_deployment() -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="remaining", response_id="resp_previous" + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("narrowing", ("affinity", "cooldown")) +@pytest.mark.parametrize("tags", (("component:chat-search",), ("&vision",), ("!region:eu",))) +@pytest.mark.parametrize("route", ("model", "routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_survives_deployment_narrowing( + narrowing: Literal["affinity", "cooldown"], tags: tuple[str, ...], route: str +) -> None: + router: Final = _fail_open_router(route=route) + requested_model: Final = "chat" if route == "model" else "chat-test" if route == "wildcard" else "public-chat" + with patch("time.time", return_value=1_700_000_000.0): + if narrowing == "cooldown": + router.cooldown_cache.add_deployment_to_cooldown( + model_id="opted-in", + original_exception=RuntimeError("unavailable"), + exception_status=500, + cooldown_time=60, + ) + candidates: Final = await router.async_get_healthy_deployments( + model=requested_model, + request_kwargs={ + "litellm_metadata": {"tags": list(tags), "inherited_tags": [], "user_api_key_team_id": "team-a"}, + **( + {"previous_response_id": _response_id_for_remaining_deployment()} if narrowing == "affinity" else {} + ), + }, + ) + assert isinstance(candidates, (list, tuple)) + assert tuple(deployment["model_info"]["id"] for deployment in candidates) == ("remaining",) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("route", ("routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_resolves_public_model_names(route: str) -> None: + router: Final = _fail_open_router(route=route) + candidates: Final = await router.async_get_healthy_deployments( + model="chat-test" if route == "wildcard" else "public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-a"}, + }, + ) + assert isinstance(candidates, (list, tuple)) + assert {deployment["model_info"]["id"] for deployment in candidates} == {"opted-in", "remaining"} + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("opted_in_group", (None, "other-group")) +async def test_tag_fail_open_requires_opt_in_in_the_requested_group(opted_in_group: str | None) -> None: + router: Final = _fail_open_router(opted_in_group) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": {"tags": ["component:chat-search"], "inherited_tags": []}, + }, + ) + + +@pytest.mark.asyncio() +async def test_tag_fail_open_does_not_use_another_teams_opt_in() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused"}, + "model_info": { + "id": team_id, + "team_id": team_id, + "team_public_model_name": "public-chat", + "allow_fail_open": team_id == "team-a", + }, + } + for team_id in ("team-a", "team-b") + ], + enable_tag_filtering=True, + ) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-b"}, + }, + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("inherited_tag", ("&vision", "!region:eu")) +async def test_tag_fail_open_preserves_inherited_policy_after_affinity(inherited_tag: str) -> None: + router: Final = _fail_open_router() + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": { + "tags": [inherited_tag, "component:chat-search"], + "inherited_tags": [inherited_tag], + }, + }, + ) + + @pytest.mark.asyncio() async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool(): router = litellm.Router( From 79c6f14f4cb98a78c82e2aef7e71a759d48a7af5 Mon Sep 17 00:00:00 2001 From: qtim0 <154377292+qtim0@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:23:56 -0800 Subject: [PATCH 03/18] fix(audio-transcription): keep transcripts out of transcription error messages (#50137) Backport of #50137 to lts/1.108.x. Cherry-picked from 5085022580d (main). Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../convert_dict_to_response.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 4 +- .../audio_transcription/transformation.py | 2 +- .../transcriptions/whisper_transformation.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../gemini_transcribe_transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 7 +- .../xai/audio_transcription/transformation.py | 2 +- .../types/llms/gemini_audio_transcription.py | 2 +- .../llms/vertex_ai_gemini_transcription.py | 2 +- .../types/llms/vertex_ai_speech_to_text.py | 2 + litellm/types/utils.py | 2 + .../test_convert_dict_to_response.py | 18 +++++ ...gram_audio_transcription_transformation.py | 30 +++++++- .../test_transformation.py | 35 ++++++--- ...mini_audio_transcription_transformation.py | 77 +++++++++++++++++++ ...tral_audio_transcription_transformation.py | 22 ++++++ .../test_whisper_transformation.py | 9 +++ ...loud_audio_transcription_transformation.py | 10 +++ ...eway_audio_transcription_transformation.py | 11 +++ ...x_ai_audio_transcription_transformation.py | 14 +++- ...tex_ai_gemini_transcribe_transformation.py | 15 ++++ ...sonx_audio_transcription_transformation.py | 26 +++++-- ..._xai_audio_transcription_transformation.py | 19 +++++ 28 files changed, 288 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8b94c34942c..3d2b5c65dff 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -995,9 +995,10 @@ def convert_to_model_response_object( if isinstance(e, APIError): raise + hide_reply: Final = response_type == "audio_transcription" received_args: Final = dict( - response_object=response_object, - model_response_object=model_response_object, + response_object="" if hide_reply else response_object, + model_response_object="" if hide_reply else model_response_object, response_type=response_type, stream=stream, start_time=start_time, diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index a3812dac8ef..c2ac3d0d5fb 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -128,7 +128,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}") from None def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 04d2717d64f..100d2c2fb02 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -152,7 +152,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}") from None def get_complete_url( self, diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 48a345a4940..5d3b5e96e0b 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -136,13 +136,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): except ValueError: raise GeminiError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}", + message="Received non-JSON response from Gemini Interactions API", ) parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json) if parsed.status != "completed": raise GeminiError( status_code=raw_response.status_code, - message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}", + message=f"Gemini transcription interaction did not complete (status={parsed.status})", ) text_contents: Final = tuple( content diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 4fce7a95350..a17a572854c 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise MistralAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Mistral audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index 8f96f688189..50da4e57dad 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -140,7 +140,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if any(key in raw_response_json for key in TranscriptionResponse.model_fields): return TranscriptionResponse(**raw_response_json) else: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - raw_response_json, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index 3ae5f6fa69c..4cc05753eed 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise OVHCloudException( - message=raw_response.text, + message="Received non-JSON response from OVHCloud audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 7ebf4475355..ce63452a45a 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise ScalewayAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Scaleway audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index f4db5eb110c..23476e0e52e 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -180,7 +180,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}", + message="Received non-JSON response from Vertex AI Gemini transcription", ) parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json) texts: Final = tuple( diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 5603be81095..cf909787e2e 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -171,7 +171,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + message="Received non-JSON response from Google Speech-to-Text", ) parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json) transcripts: Final = tuple( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 77ef03b8e10..4ec112c42a8 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -174,7 +174,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json: Final = raw_response.json() except Exception as e: - raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() @@ -190,10 +190,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran response_kwargs["usage"] = usage if not response_kwargs: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - response_object, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") response: Final = TranscriptionResponse(**response_kwargs) diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e34034b5ae9..ca8b6837b94 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -38,7 +38,7 @@ class _XAISttWord(LiteLLMBaseModel): class _XAISttResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", hide_input_in_errors=True) text: str = "" language: str = "unknown" duration: float | None = None diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index c1cf3452945..59dd15aa13e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel): class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) id: str | None = None status: str | None = None diff --git a/litellm/types/llms/vertex_ai_gemini_transcription.py b/litellm/types/llms/vertex_ai_gemini_transcription.py index e78aafaca9f..d7bc136bcd1 100644 --- a/litellm/types/llms/vertex_ai_gemini_transcription.py +++ b/litellm/types/llms/vertex_ai_gemini_transcription.py @@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel): class VertexGeminiTranscriptionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = () usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None diff --git a/litellm/types/llms/vertex_ai_speech_to_text.py b/litellm/types/llms/vertex_ai_speech_to_text.py index f3d343d13ce..3dea7610150 100644 --- a/litellm/types/llms/vertex_ai_speech_to_text.py +++ b/litellm/types/llms/vertex_ai_speech_to_text.py @@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel): class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel): + model_config = ConfigDict(hide_input_in_errors=True) + results: list[VertexSpeechToTextResult] = Field(default=[]) metadata: VertexSpeechToTextResponseMetadata | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 104ab9d928d..77160e0adf1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2848,6 +2848,8 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): class TranscriptionResponse(OpenAIObject): + model_config = ConfigDict(hide_input_in_errors=True) + text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 0c601a6b2c8..4b341562709 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2695,6 +2695,24 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + def test_parse_error_keeps_the_transcript_out_of_the_exception(self): + import traceback + + from litellm.types.utils import TranscriptionResponse + + transcript: Final = "CANARYTRANSCRIPT customer account ending 2750" + + with pytest.raises(Exception, match="Invalid response object") as exc_info: + convert_to_model_response_object( + response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}}, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + rendered: Final = "".join(traceback.format_exception(exc_info.value)) + assert "received_args=" in rendered + assert "CANARYTRANSCRIPT" not in rendered + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index aff90549a7b..d298142fe4b 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,10 +1,13 @@ import io import os import pathlib +import traceback +from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData from litellm.llms.deepgram.audio_transcription.transformation import ( @@ -587,10 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object, [{"word": "hello", "start": 0.0}], ], ) -def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object): +def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object): raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {})) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) + + +@pytest.mark.parametrize( + ("payload", "failing_lookup_error"), + [ + ({"results": None}, TypeError), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError), + ({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError), + (_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError), + ], +) +def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body( + payload: object, failing_lookup_error: type[Exception] +): + raw_response: Final = httpx.Response(200, json=payload) + + with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: + DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, failing_lookup_error) + assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}" + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index 86252a1a95d..701327263b8 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -1,9 +1,11 @@ +import traceback from collections.abc import Iterator from typing import Final import httpx import pytest import respx +from pydantic import ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache @@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li @pytest.mark.parametrize( "payload", [ - ["not", "an", "object"], - "plain text", - 7, - {"text": ["not", "text"]}, - {"text": "t", "words": None}, - {"text": "t", "words": 7}, - {"text": "t", "words": "not a list"}, - {"text": "t", "words": ["not an object"]}, - {"text": "t", "words": [{"type": "word", "text": "ok"}, None]}, + ["sensitive", "an", "object"], + "sensitive text", + {"text": ["sensitive", "text"]}, + {"text": "sensitive", "words": None}, + {"text": "sensitive", "words": 7}, + {"text": "sensitive", "words": "not a list"}, + {"text": "sensitive", "words": ["not an object"]}, + {"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]}, ], ) -def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object): +def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object): raw_response = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) + + +def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload(): + raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]}) + + with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: + ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, ValidationError) + assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}" + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) class TestElevenLabsAudioTranscription: diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..a9ef468525f 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,8 +1,10 @@ import base64 import json +import traceback import httpx import pytest +import respx import litellm from litellm.llms.gemini.audio_transcription.transformation import ( @@ -241,6 +243,21 @@ class TestTransformResponse: with pytest.raises(GeminiError, match="non-JSON"): config.transform_audio_transcription_response(raw) + def test_non_completed_status_error_omits_response_body(self, config): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = "incomplete" + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(make_response(payload)) + assert "status=incomplete" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + def test_non_json_error_omits_response_body(self, config): + raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test")) + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(raw) + assert "sensitive body" not in str(excinfo.value) + def test_word_without_offsets_survives(self, config): payload = json.loads(json.dumps(COMPLETED_RESPONSE)) payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}] @@ -249,6 +266,66 @@ class TestTransformResponse: assert response.get("duration") is None + +class TestNonCompletedInteractionThroughRouter: + @pytest.fixture + def router(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return litellm.Router( + model_list=[ + { + "model_name": "transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"}, + } + ], + num_retries=2, + ) + + def _interaction(self, status): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = status + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + return payload + + @pytest.mark.asyncio + @respx.mock + async def test_retry_recovers_an_incomplete_interaction(self, router): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + side_effect=[ + httpx.Response(200, json=self._interaction("incomplete")), + httpx.Response(200, json=COMPLETED_RESPONSE), + ] + ) + response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 2 + assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"] + + @pytest.mark.asyncio + @respx.mock + @pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None]) + async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=self._interaction(status)) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 3 + assert f"status={status}" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + @pytest.mark.asyncio + @respx.mock + async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router): + payload = self._interaction("incomplete") + payload["steps"][0]["content"] = "sensitive transcript text" + respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=payload) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value)) + + class TestSubtitleSynthesisThroughHandler: def _transform(self, config, response_format): from unittest.mock import Mock diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..494b1fe34ad 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -3,12 +3,15 @@ from unittest.mock import MagicMock import httpx import litellm +import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, + MistralAudioTranscriptionException, ) from litellm.types.utils import TranscriptionResponse from litellm.utils import ProviderConfigManager @@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + +@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}]) +def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object): + raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]}) + + with pytest.raises(ValidationError) as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py index 2dc24b1b313..339d62ef7ea 100644 --- a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py +++ b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py @@ -5,8 +5,10 @@ and transform_audio_transcription_response. import io import json +import traceback from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.openai.transcriptions.whisper_transformation import ( @@ -104,3 +106,10 @@ class TestWhisperTransformResponse: is_json=False, ) ) + + def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self): + config = OpenAIWhisperAudioTranscriptionConfig() + raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]}) + with pytest.raises(ValueError, match="Invalid response format") as exc_info: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 422658932e6..ed8ec88e209 100644 --- a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig +from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.types.utils import TranscriptionResponse @@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa _transform(payload) assert "spoken secret" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "spoken secret') + + with pytest.raises(OVHCloudException, match="non-JSON") as exc_info: + OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "spoken secret" not in str(exc_info.value) diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index f313f8bd873..16e60a15539 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json(): ) +def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response( + 200, text='{"text": "sensitive', headers={"content-type": "application/json"} + ) + + with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info: + ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type(): """When Scaleway responds with text/srt/vtt (response_format="text" etc.), the content-type is not application/json — return the body as plain text diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 5898d933941..02b7c0ae71a 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx import pytest - +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( @@ -208,6 +208,18 @@ class TestTransformResponse: response = config.transform_audio_transcription_response(raw_response) assert response.text == "" + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive') + with pytest.raises(VertexAIError, match="non-JSON") as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"}) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + def test_fractional_billed_duration(self, config): raw_response = httpx.Response( status_code=200, diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index 82ea034f91b..532e013c3d7 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -4,6 +4,7 @@ import os import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import ( @@ -255,6 +256,20 @@ class TestTransformResponse: with pytest.raises(VertexAIError, match="non-JSON"): config.transform_audio_transcription_response(raw_response) + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive') + with pytest.raises(VertexAIError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response( + status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]} + ) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + class TestValidateEnvironment: def test_sets_oauth_headers(self): diff --git a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index b6e5007c456..c9f6f566ad6 100644 --- a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription. Validates the WatsonX transcription response transformation. """ +import traceback from unittest.mock import MagicMock import httpx @@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model( assert {key: response[key] for key in expected_extras} == expected_extras -@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}]) -def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body( +@pytest.mark.parametrize( + "payload", + [ + {"model": "sensitive"}, + {"duration": 2.0, "segments": [{"text": "sensitive"}]}, + {"text": None, "usage": None, "note": "sensitive"}, + ], +) +def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error( payload: dict[str, object], ) -> None: with pytest.raises(ValueError, match="Invalid response format") as exc_info: _transform_transcription_response(payload) - assert exc_info.value.args == ( - "Invalid response format. Received response does not match the expected format. Got: ", - payload, - ) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True]) @@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou _transform_transcription_response(payload) assert "input_value" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None: + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(ValueError, match="Error transforming response to json") as exc_info: + IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py index e2e3fc3d4dc..3505116819b 100644 --- a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py +++ b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py @@ -1,3 +1,5 @@ +import traceback + import httpx import pytest @@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status(): assert "Incorrect API key provided" in exc.value.message +@pytest.mark.parametrize( + "content", + [ + b'{"text": ["sensitive"]}', + b'{"text": "ok", "words": [{"text": ["sensitive"]}]}', + b'{"text": "sensitive', + ], +) +def test_transform_response_keeps_a_malformed_body_out_of_the_error(content): + raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt")) + + with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc: + CONFIG.transform_audio_transcription_response(raw_response=raw) + + assert "sensitive" not in "".join(traceback.format_exception(exc.value)) + + def test_transcription_routes_to_xai_stt(monkeypatch): monkeypatch.delenv("XAI_API_KEY", raising=False) monkeypatch.setattr(litellm, "xai_key", None) From 6ea6ae380ed4c5895f57ac0cece658d4dd63985c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 02:34:09 +0000 Subject: [PATCH 04/18] fix(proxy): treat Postgres restart and shutdown errors as DB unavailable, not poison spend-log rows (#50132) Backport of #50132 to lts/1.108.x. The line had no _prisma_error_code helper (main got it in #49190), so it is added here verbatim for db_write_never_sent. Cherry-picked from b080c59e78e25d73934ad8550e832d3ee7cd882d (main). Conflict in tests/unit/proxy/db/test_exception_handler.py resolved by keeping only the tests this PR adds, the two neighbouring tests from #49190 are not on this line. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../tool_discovery_queue.py | 7 + litellm/proxy/db/exception_handler.py | 86 +++++++++++ litellm/proxy/db/tool_registry_writer.py | 21 ++- litellm/proxy/utils.py | 27 +++- .../test_tool_discovery_queue.py | 17 ++- tests/unit/proxy/db/test_exception_handler.py | 136 +++++++++++++++++- .../proxy/db/test_tool_registry_writer.py | 51 +++++-- .../prisma_and_spend/test_spend_functions.py | 128 ++++++++++++++++- 8 files changed, 451 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py index e9ec040023c..abbeab7f89d 100644 --- a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py +++ b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py @@ -55,3 +55,10 @@ class ToolDiscoveryQueue: """Return and clear all pending rows.""" items, self._pending = list(self._pending.values()), {} return items + + def requeue(self, items: Iterable[ToolDiscoveryQueueItem]) -> None: + """Put rows from a failed write back, ahead of and merged with anything queued since the flush.""" + since_flush: Final = self._pending + self._pending = {} + for item in (*items, *since_flush.values()): + self.add_update(item) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 669df7454e3..23d83829d8c 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -31,6 +31,24 @@ _CONNECTION_CAPACITY_PHRASES: Final = ( "too many connections for database", "remaining connection slots are reserved", ) +_NOT_ACCEPTING_CONNECTIONS_SQLSTATE: Final = "57P03" +_NOT_ACCEPTING_CONNECTIONS_PHRASES: Final = ( + "the database system is starting up", + "the database system is shutting down", + "the database system is not yet accepting connections", + "the database system is not accepting connections", + "the database system is in recovery mode", +) +_SERVER_TERMINATED_SQLSTATES: Final = frozenset({"57P01", "57P02", "57P05"}) +_SERVER_TERMINATED_PHRASES: Final = ( + "terminating connection due to administrator command", + "terminating connection due to immediate shutdown command", + "terminating connection because of unexpected sigquit signal", + "terminating connection because of crash of another server process", + "terminating connection due to idle-session timeout", +) +_UNREACHABLE_PRISMA_CODE: Final = "P1001" +_UNREACHABLE_PHRASES: Final = ("can't reach database server", "cannot reach database server") _PRISMA_POOL_TIMEOUT_CODE: Final = "P2024" _LOCK_TIMEOUT_SQLSTATE: Final = "55P03" _LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout" @@ -74,6 +92,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) +def _prisma_error_code(e: Exception) -> str | None: + import prisma.engine.errors + + if isinstance(e, _exception_types(prisma.errors.DataError)): + code: Final[object] = getattr(e, "code", None) + return code if isinstance(code, str) else None + if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)): + return None + try: + body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2]) + except ValidationError: + return None + engine_code: Final = body.get("error_code") + return engine_code if isinstance(engine_code, str) else None + + class PrismaDBExceptionHandler: """ Class to handle DB Exceptions or Connection Errors @@ -123,6 +157,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection @staticmethod @@ -271,6 +307,39 @@ class PrismaDBExceptionHandler: error_message: Final = str(e).lower() return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES) + @staticmethod + def is_database_not_accepting_connections_error(e: Exception) -> bool: + """True iff Postgres refused the connection because it is starting up, shutting + down or in crash recovery (SQLSTATE 57P03). The refusal comes before any statement + runs, so nothing the write carried was applied.""" + import prisma + + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) == _NOT_ACCEPTING_CONNECTIONS_SQLSTATE: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _NOT_ACCEPTING_CONNECTIONS_PHRASES) + + @staticmethod + def is_database_lifecycle_error(e: Exception) -> bool: + """True iff Postgres failed the statement because of a server lifecycle event: + a refused connection during startup, shutdown or recovery (57P03), or a session + the server terminated (57P01, 57P02, 57P05). The server comes back on its own, + so the rows are not at fault. A terminated session may have committed before it + was cut, so only ``is_database_not_accepting_connections_error`` proves a + non-idempotent write never applied.""" + import prisma + + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) in _SERVER_TERMINATED_SQLSTATES: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _SERVER_TERMINATED_PHRASES) + @staticmethod def is_lock_timeout_error(e: Exception) -> bool: """True iff Postgres cancelled a statement for exceeding ``lock_timeout`` @@ -388,6 +457,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_lifecycle_error(e): + return True if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e): return True if "cached plan must not change result type" in str(e).lower(): @@ -662,3 +733,18 @@ def db_write_failure_is_requeue_safe(e: Exception) -> bool: return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES + + +def db_write_never_sent(e: Exception) -> bool: + """Whether the database could not take the write at all, because it was unreachable (P1001), + refusing connections (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again.""" + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if PrismaDBExceptionHandler.is_database_capacity_error(e): + return True + if _prisma_error_code(e) == _UNREACHABLE_PRISMA_CODE: + return True + if not PrismaDBExceptionHandler.is_prisma_error(e): + return False + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _UNREACHABLE_PHRASES) diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index f7ec88945a1..9600fb40ad3 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -23,7 +23,11 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, aggregate_tool_discoveries, ) -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe +from litellm.proxy.db.exception_handler import ( + call_with_db_reconnect_retry, + db_write_failure_is_requeue_safe, + db_write_never_sent, +) from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions @@ -134,8 +138,19 @@ async def flush_tool_discovery_queue(prisma_client: "PrismaClient", tool_discove return try: await batch_upsert_tools(prisma_client=prisma_client, items=items) - except Exception as e: # noqa: BLE001 # a lost tool row only delays its discovery, the scheduler job must not fail - verbose_proxy_logger.debug("flush_tool_discovery_queue error (non-blocking): %s", e) + except Exception as e: # noqa: BLE001 # the rows go back to the queue or are dropped, the scheduler job must not fail + if db_write_never_sent(e): + tool_discovery_queue.requeue(items) + verbose_proxy_logger.warning( + "Tool discovery - %d tool rows went back to the queue for the next tick. Error: %s", len(items), str(e) + ) + return + verbose_proxy_logger.error( + "Tool discovery - dropped %d tool rows: the failed statement may have applied or the database " + "refused the data, so re-sending it is not safe. Error: %s", + len(items), + str(e), + ) async def commit_tool_discoveries( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 486028297b3..8479f8e45b9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -198,6 +198,7 @@ from litellm.proxy.db.db_url_settings import ( from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, + db_write_never_sent, ) from litellm.proxy.db.health_check_latest import ( LatestHealthCheckRow, @@ -8500,9 +8501,20 @@ async def update_daily_tag_spend( def _rollup_batch_never_applied(e: Exception) -> bool: """True when a drained rollup batch provably never reached its rows: Postgres had no - connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row - lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued.""" - return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e) + connection for it (53300), was unreachable or refusing connections (P1001, 57P03), or + cancelled it under ``lock_timeout`` before it took the row lock (55P03). In each case + re-sending the batch cannot double-count, so it is requeued.""" + return ( + PrismaDBExceptionHandler.is_database_capacity_error(e) + or db_write_never_sent(e) + or PrismaDBExceptionHandler.is_lock_timeout_error(e) + ) + + +def _rollup_failure_stops_job(e: Exception) -> bool: + """True when the database cannot take any write, so the job stops instead of draining + the queues after this one into the same failure.""" + return PrismaDBExceptionHandler.is_database_capacity_error(e) or db_write_never_sent(e) async def update_spend_logs_job( @@ -8593,12 +8605,12 @@ async def _run_spend_logs_job( *prisma_client.tool_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during tool usage " + "Spend tracking - database unavailable or rollup row lock timed out during tool usage " "flush, requeued %s tool usage transactions for the next flush: %s", len(tool_usage_to_process), tool_tracking_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err): + if _rollup_failure_stops_job(tool_tracking_err): raise else: verbose_proxy_logger.error( @@ -8625,12 +8637,12 @@ async def _run_spend_logs_job( *prisma_client.model_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during model usage " + "Spend tracking - database unavailable or rollup row lock timed out during model usage " "flush, requeued %s model usage transactions for the next flush: %s", len(model_usage_to_process), model_usage_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err): + if _rollup_failure_stops_job(model_usage_err): raise else: verbose_proxy_logger.error( @@ -8807,6 +8819,7 @@ def _is_transient_spend_log_write_error(e: Exception) -> bool: return ( PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) + or PrismaDBExceptionHandler.is_database_lifecycle_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e) ) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py index 0a015255f2e..814fdcb4eda 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py @@ -2,7 +2,6 @@ Unit tests for ToolDiscoveryQueue. """ - import pytest from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( @@ -63,3 +62,19 @@ def test_aggregate_sums_call_counts_across_workers_flushes(): {"tool_name": "get_weather", "origin": "user_defined", "call_count": 5}, {"tool_name": "search", "call_count": 1}, ) + + +def test_requeued_rows_go_ahead_of_and_merge_with_sightings_since_the_flush(queue: ToolDiscoveryQueue) -> None: + queue.add_update({"tool_name": "get_weather", "origin": "mcp_server"}) + queue.add_update({"tool_name": "search", "origin": "user_defined"}) + failed_write = queue.flush() + queue.add_update({"tool_name": "translate", "origin": "user_defined"}) + queue.add_update({"tool_name": "search", "origin": "user_defined"}) + + queue.requeue(failed_write) + + assert queue.flush() == [ + {"tool_name": "get_weather", "origin": "mcp_server", "call_count": 1}, + {"tool_name": "search", "origin": "user_defined", "call_count": 2}, + {"tool_name": "translate", "origin": "user_defined", "call_count": 1}, + ] diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index a4c16c0949b..6bf5f6b94eb 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -22,7 +22,7 @@ from prisma.errors import ( import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException -from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_never_sent # Test is_database_connection_error method @@ -966,3 +966,137 @@ def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None ) def test_is_statement_timeout_error(error: Exception, expected: bool) -> None: assert PrismaDBExceptionHandler.is_statement_timeout_error(error) is expected + + +def _connector_error(message: str) -> DataError: + return DataError( + data={"user_facing_error": {"message": f"Error in connector: Error querying the database: FATAL: {message}"}} + ) + + +def _batched_postgres_error(sqlstate: str, message: str) -> DataError: + return DataError( + data={ + "user_facing_error": { + "message": "Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: " + f'None, kind: QueryError(PostgresError {{ code: "{sqlstate}", message: "{message}", severity: "FATAL" }}) }})' + } + } + ) + + +def _localized_error(sqlstate: str) -> DataError: + return DataError(data={"user_facing_error": {"message": "FATAL: Datenbank", "meta": {"code": sqlstate}}}) + + +_NOT_ACCEPTING_CONNECTIONS: Final = ( + _connector_error("the database system is starting up"), + _connector_error("the database system is shutting down"), + _connector_error("the database system is not yet accepting connections"), + _connector_error("the database system is not accepting connections"), + _connector_error("the database system is in recovery mode"), + _localized_error("57P03"), + _batched_postgres_error("57P03", "the database system is starting up"), + EngineRequestError( + MagicMock(status=500), + '{"is_panic":false,"message":"Error in connector: Error querying the database: FATAL: the database system is ' + 'starting up","backtrace":null}', + ), + RawQueryError( + data={ + "user_facing_error": { + "message": "Raw query failed. Code: `57P03`. Message: `db error: FATAL: the database system is starting up`", + "meta": {"code": "57P03", "message": "FATAL: the database system is starting up"}, + "error_code": "P2010", + } + } + ), +) +_SERVER_TERMINATED: Final = ( + _connector_error("terminating connection due to administrator command"), + _connector_error("terminating connection due to immediate shutdown command"), + _connector_error("terminating connection because of unexpected SIGQUIT signal"), + _connector_error("terminating connection because of crash of another server process"), + _connector_error("terminating connection due to idle-session timeout"), + _localized_error("57P01"), + _localized_error("57P02"), + _localized_error("57P05"), + _batched_postgres_error("57P01", "terminating connection due to administrator command"), +) + + +@pytest.mark.parametrize( + ("error", "connection_error"), + [ + *((error, True) for error in _NOT_ACCEPTING_CONNECTIONS), + *((error, False) for error in _SERVER_TERMINATED), + ], +) +def test_postgres_lifecycle_errors_mean_the_database_is_unavailable(error: Exception, connection_error: bool) -> None: + """A server that is restarting, shutting down or cutting sessions is a transient outage: + auth answers 503 rather than 401, and the spend-log writer must not take the rows for poison. + Only a server refusing connections opens the ``allow_requests_on_db_unavailable`` fallback, + one terminated session does not prove the database is down.""" + assert { + "lifecycle": PrismaDBExceptionHandler.is_database_lifecycle_error(error), + "service_unavailable": PrismaDBExceptionHandler.is_database_service_unavailable_error(error), + "connection_error": PrismaDBExceptionHandler.is_database_connection_error(error), + "permanent": PrismaDBExceptionHandler.is_permanent_database_fault(error), + } == { + "lifecycle": True, + "service_unavailable": True, + "connection_error": connection_error, + "permanent": False, + } + + +@pytest.mark.parametrize( + ("error", "never_sent"), + [ + *((error, True) for error in _NOT_ACCEPTING_CONNECTIONS), + *((error, False) for error in _SERVER_TERMINATED), + (DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}}), True), + (DataError(data={"user_facing_error": {"error_code": "P1001", "message": "unreachable"}}), True), + (_capacity_error("sorry, too many clients already"), True), + (_localized_error("53300"), True), + ( + DataError( + data={ + "user_facing_error": { + "error_code": "P2024", + "message": "Timed out fetching a new connection from the connection pool", + } + } + ), + True, + ), + (httpx.ReadTimeout("read timed out"), False), + (_localized_error("57014"), False), + ], +) +def test_db_write_never_sent_only_for_failures_before_any_statement_ran(error: Exception, never_sent: bool) -> None: + """A refused, unreachable or full server provably applied nothing, while a terminated session + or a timeout may have committed before the failure, so only the former lets a non-idempotent + write be replayed.""" + assert db_write_never_sent(error) is never_sent + + +@pytest.mark.parametrize( + "error", + [ + _localized_error("57P04"), + _localized_error("57014"), + _localized_error("23505"), + DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}), + RuntimeError("the database system is starting up"), + ], +) +def test_other_failures_are_not_postgres_lifecycle_errors(error: Exception) -> None: + assert PrismaDBExceptionHandler.is_database_lifecycle_error(error) is False + assert db_write_never_sent(error) is False + + +def test_lifecycle_error_is_seen_through_a_wrapping_exception() -> None: + wrapped: Final = RuntimeError("spend flush failed") + wrapped.__cause__ = _connector_error("the database system is starting up") + assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True diff --git a/tests/unit/proxy/db/test_tool_registry_writer.py b/tests/unit/proxy/db/test_tool_registry_writer.py index 76898b0eeab..a5ad897913e 100644 --- a/tests/unit/proxy/db/test_tool_registry_writer.py +++ b/tests/unit/proxy/db/test_tool_registry_writer.py @@ -69,9 +69,7 @@ def _make_prisma( prisma.db.litellm_tooltable = MagicMock() prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return) prisma.db.execute_raw = AsyncMock(return_value=1) - prisma.db.litellm_tooltable.find_many = AsyncMock( - return_value=find_many_rows if find_many_rows is not None else [] - ) + prisma.db.litellm_tooltable.find_many = AsyncMock(return_value=find_many_rows if find_many_rows is not None else []) prisma.db.litellm_tooltable.find_unique = AsyncMock(return_value=find_unique_row) prisma.db.litellm_verificationtoken = MagicMock() prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(key_rows)) @@ -227,9 +225,7 @@ async def test_get_tool_found(): result = await get_tool(prisma, "my_tool") assert result is not None assert result.tool_name == "my_tool" - prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with( - where={"tool_name": "my_tool"} - ) + prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(where={"tool_name": "my_tool"}) @pytest.mark.asyncio @@ -266,9 +262,7 @@ async def test_update_tool_policy_calls_upsert_then_get_tool(): updated_by="admin", ) prisma = _make_prisma(find_unique_row=row) - result = await update_tool_policy( - prisma, "my_tool", updated_by="admin", input_policy="blocked" - ) + result = await update_tool_policy(prisma, "my_tool", updated_by="admin", input_policy="blocked") assert result is not None assert result.input_policy == "blocked" prisma.db.litellm_tooltable.upsert.assert_awaited_once() @@ -567,3 +561,42 @@ async def test_failed_tool_table_write_requeues_the_rows_only_when_they_provably left_in_redis = [json.loads(entry) for entry in redis.lists.get(REDIS_TOOL_DISCOVERY_BUFFER_KEY, [])] expected = [[{"tool_name": "get_weather", "origin": "user_defined", "call_count": 1}]] if requeued else [] assert left_in_redis == expected, f"requeued={requeued} but redis holds {left_in_redis}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "requeued"), + [ + (_postgres_rejection("57P03"), True), + (_postgres_rejection("53300"), True), + (_postgres_rejection("57P01"), False), + (_postgres_rejection("54000"), False), + (_postgres_rejection("23502"), False), + ], + ids=[ + "restarting_server_keeps_the_rows", + "full_server_keeps_the_rows", + "terminated_session_may_have_counted_so_the_rows_are_dropped", + "oversized_value_is_dropped", + "rejected_row_is_dropped", + ], +) +async def test_failed_write_without_redis_keeps_the_rows_in_the_queue_only_when_they_provably_did_not_apply( + failure: Exception, requeued: bool +): + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.execute_raw = AsyncMock(side_effect=failure) + + await commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=queue, + redis_update_buffer=RedisUpdateBuffer(redis_cache=_InMemoryRedis()), + pod_lock_manager=PodLockManager(redis_cache=_InMemoryRedis()), + use_redis_buffer=False, + ) + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + + expected = [{"tool_name": "get_weather", "origin": "user_defined", "call_count": 2 if requeued else 1}] + assert queue.flush() == expected diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py index fc7ae531c34..293f2d49f43 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -23,9 +23,9 @@ import pytest from litellm.constants import REDIS_SPEND_LOGS_BUFFER_KEY from litellm.proxy.utils import ( MAX_SPEND_LOG_DRAIN_ITERATIONS, + drain_spend_logs_queue, monitor_spend_logs_queue, raise_failed_update_spend_exception, - drain_spend_logs_queue, recover_parked_spend_logs, update_daily_tag_spend, update_spend, @@ -985,6 +985,85 @@ async def test_update_spend_logs_job_requeues_whole_batch_when_postgres_is_out_o } == {"create_many_calls": 1, "queue_after": ["r1", "r2", "r3"], "backoff_sleeps": []} +def _postgres_restarting() -> Exception: + """prisma's shape for Postgres SQLSTATE 57P03 while the server runs crash recovery after + a restart: a base ``DataError`` carrying only the connector message.""" + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error in connector: Error querying the database: FATAL: the database system is starting up", + "backtrace": None, + } + } + ) + + +def _postgres_unreachable() -> Exception: + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Can't reach database server at `db.internal`:`5432`", + "error_code": "P1001", + } + } + ) + + +def _postgres_terminated_session() -> Exception: + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error in connector: Error querying the database: FATAL: terminating connection due to " + "administrator command", + "meta": {"code": "57P01", "message": "terminating connection due to administrator command"}, + } + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [_postgres_restarting(), _postgres_terminated_session()], ids=["57P03", "57P01"]) +async def test_update_spend_logs_job_keeps_every_row_while_postgres_restarts( + error: Exception, + mock_prisma_client: MagicMock, + make_spend_log_row: Callable[..., dict[str, object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A restarting server refuses or cuts every statement, so the batch is not bisected + into per-row drops: it is retried whole and then goes back to the queue head.""" + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + proxy_logging: Final = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [ + make_spend_log_row(request_id="r1"), + make_spend_log_row(request_id="r2"), + make_spend_log_row(request_id="r3"), + make_spend_log_row(request_id="r4"), + ] + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=error) + + with suppress(Exception): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert { + "create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count, + "queue_after": [row["request_id"] for row in mock_prisma_client.spend_log_transactions], + } == {"create_many_calls": 4, "queue_after": ["r1", "r2", "r3", "r4"]} + + def _tool_usage_transaction(request_id: str) -> object: from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -1177,6 +1256,53 @@ async def test_model_usage_flush_requeues_and_stops_the_job_when_postgres_is_out } == {"queue_after": [first, second], "autorouter_queue_after": 1} +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [_postgres_restarting(), _postgres_unreachable()], ids=["57P03", "P1001"]) +async def test_rollups_requeue_and_stop_the_job_while_postgres_cannot_take_writes( + error: Exception, + mock_prisma_client: MagicMock, +) -> None: + """A server that is restarting or unreachable never saw the rollup batch, so the tool + usage increments are kept for the next flush, and the job stops before draining the + auto-router queue into the same outage.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=error) + mock_prisma_client.spend_log_transactions = [] + first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2") + mock_prisma_client.tool_usage_transactions = [first, second] + mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")] + + with suppress(Exception): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert { + "queue_after": mock_prisma_client.tool_usage_transactions, + "autorouter_queue_after": len(mock_prisma_client.autorouter_turn_transactions), + } == {"queue_after": [first, second], "autorouter_queue_after": 1} + + +@pytest.mark.asyncio +async def test_rollups_drop_a_batch_whose_session_the_server_terminated( + mock_prisma_client: MagicMock, +) -> None: + """A terminated session (57P01) may have committed before it was cut, and the rollup + increments are not idempotent, so the batch is dropped rather than counted twice.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=_postgres_terminated_session()) + mock_prisma_client.spend_log_transactions = [] + mock_prisma_client.tool_usage_transactions = [_tool_usage_transaction("r1")] + + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert mock_prisma_client.tool_usage_transactions == [] + + @pytest.mark.asyncio async def test_model_usage_flush_still_drops_a_batch_that_may_have_applied( mock_prisma_client: MagicMock, From bec75e8c371701d6876617e24249bc8e0640f6d3 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 02:35:20 +0000 Subject: [PATCH 05/18] fix(openai): never send OPENAI_ADMIN_KEY as the bearer token for a deployment without an api_key (#50218) Backport of #50218 to lts/1.108.x. Cherry-picked from f25fff42a674f6851a67c82998d1689f1423aa6f (main). Conflict in tests/unit/llms/openai/test_openai.py resolved by adding only this PR's test and imports, the neighbouring agentic-loop tests from main are not on this line. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/openai/openai.py | 7 ++++ tests/unit/llms/openai/test_openai.py | 48 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bf33070faa6..b4df0624e68 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,3 +1,4 @@ +import inspect import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping @@ -351,6 +352,11 @@ _EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str] | None) _EXTRA_QUERY_ADAPTER: Final = TypeAdapter(dict[str, object] | None) _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({}) _SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body")) +_CLIENT_API_KEY_ONLY_AUTH: Final[Mapping[str, Mapping[str, bool]]] = ( + types.MappingProxyType({"security": types.MappingProxyType({"bearer_auth": True})}) + if "security" in inspect.signature(make_request_options).parameters + else types.MappingProxyType({}) +) def _raw_chat_response( @@ -370,6 +376,7 @@ def _request_without_sdk_transform( extra_query=_EXTRA_QUERY_ADAPTER.validate_python(data.get("extra_query")), extra_body=data.get("extra_body"), timeout=timeout, + **_CLIENT_API_KEY_ONLY_AUTH, # pyright: ignore[reportArgumentType] # openai<3 make_request_options has no security kwarg ) return body, options diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 749dc823f20..961b6c88012 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -1,4 +1,6 @@ import asyncio +import contextlib +import inspect import importlib import json import os @@ -1937,3 +1939,49 @@ async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_head assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} assert sent.headers["x-caller"] == "unit" assert sent.headers["authorization"] == "Bearer sk-test" + + +_SDK_READS_OPENAI_ADMIN_KEY: Final = "admin_api_key" in inspect.signature(AsyncOpenAI.__init__).parameters + + +@pytest.mark.skipif(not _SDK_READS_OPENAI_ADMIN_KEY, reason="openai<3 has no OPENAI_ADMIN_KEY fallback to leak") +@pytest.mark.asyncio +async def test_acompletion_never_sends_the_openai_admin_key_for_a_deployment_without_a_key( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_ADMIN_KEY", "sk-admin-only") + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + outbound: Final = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + outbound.put_nowait(request.headers.get("authorization")) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-no-admin-key", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.4-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + monkeypatch.setattr(litellm, "aclient_session", http_client) + with contextlib.suppress(litellm.AuthenticationError, litellm.InternalServerError): + await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.4-mini", + api_key="", + messages=[{"role": "user", "content": "who signs this request"}], + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + sent: Final = tuple(outbound.get_nowait() for _ in range(outbound.qsize())) + assert all(authorization is None for authorization in sent), sent From d9d70c1936c659222645f3642026833b2aa36ce5 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 02:36:01 +0000 Subject: [PATCH 06/18] fix(types): define the responses compact call types for during_call guardrails and deployment hooks (#50274) Backport of #50274 to lts/1.108.x. The line also lacked the get/delete/cancel responses CallTypes members that main got in #49133, so a during_call guardrail made those routes 500 with 'aget_responses is not a valid CallTypes'. The six members are added here (CallTypes, schema.d.ts, the azure call type list), and tests/integration/conftest.py registers the shares_worker_state marker the picked integration test carries. Cherry-picked from d295f76c4ad7204de6ab388a7da2e74300ca9ef1 (main). Conflicts in litellm/types/utils.py, schema.d.ts and three test files were neighbouring main-only call types and tests, resolved by adding only the compact_responses entries and tests from this PR. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/utils.py | 10 ++ ...sponses_lifecycle_during_call_guardrail.py | 114 ++++++++++++++++++ tests/integration/conftest.py | 1 + .../llms/azure/test_azure_common_utils.py | 4 + .../test_unified_guardrail.py | 18 +++ tests/unit/test_utils.py | 23 ++++ tests/unit/types/test_utils.py | 12 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 8 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 104ab9d928d..71f634d76c9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -653,6 +653,14 @@ class CallTypes(str, Enum): retrieve_fine_tuning_job = "retrieve_fine_tuning_job" responses = "responses" aresponses = "aresponses" + get_responses = "get_responses" + aget_responses = "aget_responses" + delete_responses = "delete_responses" + adelete_responses = "adelete_responses" + cancel_responses = "cancel_responses" + acancel_responses = "acancel_responses" + compact_responses = "compact_responses" + acompact_responses = "acompact_responses" alist_input_items = "alist_input_items" llm_passthrough_route = "llm_passthrough_route" allm_passthrough_route = "allm_passthrough_route" @@ -754,6 +762,8 @@ CallTypesLiteral = Literal[ "send_message", "aresponses", "responses", + "compact_responses", + "acompact_responses", "acreate_skill", "acreate_realtime_client_secret", "arealtime_calls", diff --git a/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py new file mode 100644 index 00000000000..3227caaa7f5 --- /dev/null +++ b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py @@ -0,0 +1,114 @@ +"""GET, compact, DELETE and cancel on /v1/responses answer 200 when a during_call guardrail is on by default +(LIT-9770): the hook used to raise `'acompact_responses' is not a valid CallTypes` and the proxy returned 500""" + +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, string_value +from integration._support.process import owned_proxy +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MODEL: Final = "openai/gpt-5.4-mini" + + +def _input() -> list[JsonValue]: + return [ + {"role": "user", "content": f"Remember the number {uuid.uuid4().hex}."}, + {"role": "assistant", "content": "Got it."}, + ] + + +def _response(response_id: str) -> Reply: + body: Final[dict[str, JsonValue]] = { + "id": response_id, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _MODEL.removeprefix("openai/"), + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Got it.", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 12, "output_tokens": 5, "total_tokens": 17}, + } + return Reply(body=json.dumps(body).encode()) + + +def _deleted(response_id: str) -> Reply: + return Reply(body=json.dumps({"id": response_id, "object": "response.deleted", "deleted": True}).encode()) + + +def _guarded_config(path: Path) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + guardrail: Final[dict[str, JsonValue]] = { + "guardrail_name": f"content-filter-{uuid.uuid4().hex}", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "during_call", + "default_on": True, + "blocked_words": [{"keyword": "forbiddenword", "action": "BLOCK"}], + }, + } + path.write_text(yaml.safe_dump({**base, "guardrails": [guardrail]})) + return path + + +@pytest.fixture(scope="module") +def guarded(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("during-call-guardrail") + with owned_proxy(session_gateway, directory, {}, config=_guarded_config(directory / "config.yaml")) as candidate: + yield candidate + + +def _create(guarded: Gateway, provider: SharedProvider, upstream_id: str) -> str: + provider.expect(_response(upstream_id)) + created: Final = guarded.request("POST", "/v1/responses", {"model": _MODEL, "input": _input(), "store": True}) + assert created.status_code == 200, created.text + (upstream,) = provider.received() + assert upstream.target.endswith("/responses"), upstream.target + return string_value(_JSON_OBJECT.validate_json(created.content)["id"]) + + +def _lifecycle(guarded: Gateway, provider: SharedProvider, operation: str) -> tuple[httpx.Response, str]: + if operation == "compact": + provider.expect(_response(f"resp_{uuid.uuid4().hex}")) + return guarded.request( + "POST", "/v1/responses/compact", {"model": _MODEL, "input": _input()} + ), "/responses/compact" + upstream_id: Final = f"resp_{uuid.uuid4().hex}" + managed_id: Final = _create(guarded, provider, upstream_id) + provider.expect(_deleted(upstream_id) if operation == "delete" else _response(upstream_id)) + if operation == "get": + return guarded.request("GET", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + if operation == "delete": + return guarded.request("DELETE", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + return guarded.request("POST", f"/v1/responses/{managed_id}/cancel"), f"/responses/{upstream_id}/cancel" + + +@pytest.mark.parametrize("operation", ["get", "compact", "delete", "cancel"]) +def test_responses_lifecycle_answers_200_with_a_during_call_guardrail_on( + guarded: Gateway, provider: SharedProvider, operation: str +) -> None: + response, upstream_target = _lifecycle(guarded, provider, operation) + assert response.status_code == 200, response.text + (upstream,) = provider.received() + assert upstream.target.endswith(upstream_target), upstream.target diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e82430c3add..7b0149fd2dd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -68,6 +68,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("markers", "integration: owned real-service integration contracts") config.addinivalue_line("markers", "covers(*ids): legacy contract IDs kept for existing tests, not enforced") + config.addinivalue_line("markers", "shares_worker_state: owned proxies here share the worker DB and Redis") config.stash[REPORTS] = [] config.stash[SHARD_FILES] = _shard_files(SHARD_PATH.validate_python(config.getoption("integration_shard_files"))) config.pluginmanager.register(IntegrationReportPlugin(config)) diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index b277cac1f29..98e3b84ff24 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -456,6 +456,10 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "add_message", "arun_thread_stream", "aresponses", + "aget_responses", + "adelete_responses", + "acancel_responses", + "acompact_responses", "aresponses_websocket", "alist_input_items", "acreate_fine_tuning_job", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..1417b87417c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -3053,3 +3053,21 @@ async def test_pre_call_hook_skips_attachment_scan_when_guardrail_has_none(): ) assert len(guardrail.apply_calls) == 1 + + +@pytest.mark.asyncio +async def test_moderation_hook_lets_responses_compact_through_without_a_translation() -> None: + handler: Final = UnifiedLLMGuardrails() + guardrail: Final = RecordingGuardrail() + data: Final = { + "guardrail_to_apply": guardrail, + "model": "gpt-5.4-mini", + "input": [{"role": "user", "content": "Remember the number 42."}], + } + + result: Final = await handler.async_moderation_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), call_type="acompact_responses" + ) + + assert result is data + assert guardrail.apply_calls == [] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0a408e7c8ab..a9e3aa7da5d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -98,6 +98,7 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, async_post_call_success_deployment_hook, + async_pre_call_deployment_hook, calculate_max_parallel_requests, client, get_non_default_completion_params, @@ -10052,3 +10053,25 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ ) assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} + + +class _RecordingPreCallDeploymentHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_call_types: tuple[CallTypes | None, ...] = () + + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: + self.seen_call_types = (*self.seen_call_types, call_type) + return None + + +@pytest.mark.asyncio +async def test_pre_call_deployment_hook_hands_responses_compact_its_call_type(monkeypatch: pytest.MonkeyPatch) -> None: + hook: Final = _RecordingPreCallDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [hook]) + + await async_pre_call_deployment_hook({"model": "gpt-5.4-mini"}, "acompact_responses") + + assert hook.seen_call_types == (CallTypes.acompact_responses,) diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py index 32479c62372..bdbcbb698d4 100644 --- a/tests/unit/types/test_utils.py +++ b/tests/unit/types/test_utils.py @@ -1,7 +1,12 @@ import pytest from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_excluded_services_value -from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CallTypes, + CaptureMessageContent, + captures_span_content, +) @pytest.mark.parametrize("value", ["", " ", "auth", "none,redis", "redis,", ",postgres"]) @@ -45,3 +50,8 @@ def test_capture_message_content_constructor_accepts_any_string() -> None: ) def test_captures_span_content(mode: str | None, captures: bool) -> None: assert captures_span_content(mode) is captures + + +@pytest.mark.parametrize("call_type", ["compact_responses", "acompact_responses"]) +def test_responses_compact_call_types_round_trip_their_public_names(call_type: str) -> None: + assert CallTypes(call_type).value == call_type diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 89cc9157359..13b84896a84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28370,7 +28370,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "compact_responses" | "acompact_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ From da447bb7595acbd14841c4feb54e2a7f351db7b0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 02:37:36 +0000 Subject: [PATCH 07/18] fix(router): write the prompt caching pin when turn_off_message_logging is on (#50276) Backport of #50276 to lts/1.108.x. tests/integration/_support/redaction_probe.py also sends the standard key the picked integration test reads (main's probe got it in #47192). Cherry-picked from 3f13c1ca34874c3bfa021e6fc14ac950c56d97b5 (main). Conflicts in litellm_logging.py and the prompt caching check test were main-only neighbours (_get_completed_success_callbacks and the tools/system affinity tests), resolved by adding only this PR's method and test. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_logger.py | 5 + litellm/litellm_core_utils/litellm_logging.py | 21 +++- .../prompt_caching_deployment_check.py | 3 + tests/integration/_support/redaction_probe.py | 1 + ...ompt_cache_pin_with_message_logging_off.py | 95 +++++++++++++++++++ .../test_prompt_caching_deployment_check.py | 55 +++++++++++ 6 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 5addc994df2..9332137e3b1 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -898,6 +898,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac def redacts_messages_itself(self) -> bool: return False + def exempt_from_message_redaction(self) -> bool: + """Router pre-call checks that key routing off the request and never log it return True, so + `turn_off_message_logging` leaves the success event they receive unredacted.""" + return False + def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict: """ Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1d6250fc528..95323c2ef5e 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3751,7 +3751,11 @@ class Logging(LiteLLMLoggingBaseClass): if isinstance(callback, CustomLogger): # custom logger class from litellm.integrations.shadow_eval_logger import ShadowEvalLogger - model_call_details: dict = self.model_call_details + model_call_details: dict = ( + self._model_call_details_with_request_messages() + if callback.exempt_from_message_redaction() + else self.model_call_details + ) ################################## # call redaction hook for custom logger model_call_details = callback.redact_standard_logging_payload_from_model_call_details( @@ -4289,6 +4293,21 @@ class Logging(LiteLLMLoggingBaseClass): ) -> None: executor.submit(self.success_handler, result, start_time, end_time, cache_hit) + def _model_call_details_with_request_messages(self) -> dict[str, object]: # mutable-ok: loggers take a dict + """The success-event payload with the request messages put back where `turn_off_message_logging` + redacted them in place, for a callback that routes on the request and never logs it.""" + request_messages: Final = cast(object, self.messages) # cast-ok: the request messages are only put back as-is + standard_logging_object: Final = self.model_call_details.get("standard_logging_object") + return { + **self.model_call_details, + "messages": request_messages, + **( + {"standard_logging_object": {**standard_logging_object, "messages": request_messages}} + if isinstance(standard_logging_object, dict) + else {} + ), + } + def _should_run_sync_callbacks_for_async_calls(self) -> bool: """ Returns: diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index b2f0c658161..435f9b9c5a4 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -138,6 +138,9 @@ class PromptCachingDeploymentCheck(CustomLogger): return healthy_deployments + def exempt_from_message_redaction(self) -> bool: + return True + async def async_log_success_event( self, kwargs: _PromptCachingSuccessEventKwargs, diff --git a/tests/integration/_support/redaction_probe.py b/tests/integration/_support/redaction_probe.py index 2a344daa73c..83d1cbd9cba 100644 --- a/tests/integration/_support/redaction_probe.py +++ b/tests/integration/_support/redaction_probe.py @@ -19,6 +19,7 @@ def _payload(kwargs: Mapping[str, object], response_obj: object) -> str: "request": request, "response": response_obj.model_dump() if isinstance(response_obj, BaseModel) else response_obj, "standard_response": standard.get("response") if isinstance(standard, Mapping) else None, + "standard": standard, }, default=str, ) diff --git a/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py b/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py new file mode 100644 index 00000000000..13bf67582b3 --- /dev/null +++ b/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py @@ -0,0 +1,95 @@ +import collections +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import eventually, gateway_from_environment, object_value +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.redis_process import owned_redis +from integration._support.wire import Reply, Wire, wire_server +from integration.routing._prompt_cache_pin_support import ( + DEFAULT_MARK, + anthropic_deployment, + ask_turn, + chat_body, + claude_peer, + new_marker, + one_new_pin, + pin_config, + post_chat, + read_pins, + system_turn, +) +from pydantic import JsonValue, TypeAdapter +from redis import Redis + +from litellm.constants import REDACTED_BY_LITELLM + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 300) + +_GROUP: Final = "pin-logging-off" +_LEFT: Final = "pin-logging-off-left" +_RIGHT: Final = "pin-logging-off-right" +_FOLLOW_UPS: Final = 20 +_REDACTED_MESSAGES: Final = [{"role": "user", "content": REDACTED_BY_LITELLM}] +_LOGGED_EVENT: Final = TypeAdapter(dict[str, JsonValue]) + + +@dataclass(frozen=True, slots=True) +class _Rig: + owned: OwnedProxy + redis: "Redis[bytes]" + logger_sink: Wire + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("prompt-cache-pin-message-logging-off") + with ( + gateway_from_environment() as gateway, + owned_redis(directory) as cache, + wire_server(claude_peer) as left, + wire_server(claude_peer) as right, + wire_server(lambda request: Reply()) as logger_sink, + ): + config: Final = pin_config( + directory, + [anthropic_deployment(_GROUP, _LEFT, left.url), anthropic_deployment(_GROUP, _RIGHT, right.url)], + litellm_settings={ + "turn_off_message_logging": True, + "callbacks": ["integration._support.redaction_probe.probe"], + "cache": False, + "enable_redis_auth_cache": False, + }, + ) + environment: Final = { + "REDIS_HOST": cache.host, + "REDIS_PORT": str(cache.port), + "REDACTION_PROBE_URL": logger_sink.url, + } + with ( + owned_proxy_process(gateway, directory, environment, config=config, workers=1) as owned, + Redis(host=cache.host, port=cache.port, socket_timeout=2) as redis, + ): + yield _Rig(owned, redis, logger_sink) + + +def test_follow_ups_stay_on_the_first_deployment_while_the_logger_sees_redacted_messages(rig: _Rig) -> None: + before: Final = frozenset(read_pins(rig.redis)) + body: Final = chat_body(_GROUP, [system_turn(new_marker(), DEFAULT_MARK), ask_turn(new_marker())]) + + _, first_served = post_chat(rig.owned.gateway, body) + _, pin = one_new_pin(rig.redis, before) + assert pin.model_id == first_served, (pin, first_served) + + served: Final = tuple(post_chat(rig.owned.gateway, body)[1] for _ in range(_FOLLOW_UPS)) + assert served == (first_served,) * _FOLLOW_UPS, collections.Counter(served) + + eventually(rig.logger_sink.received.qsize, lambda count: count >= 1 + _FOLLOW_UPS, seconds=30) + logged: Final = tuple( + object_value(_LOGGED_EVENT.validate_json(request.body)["standard"])["messages"] + for request in rig.logger_sink.drain() + ) + assert len(logged) == 1 + _FOLLOW_UPS, len(logged) + assert all(messages == _REDACTED_MESSAGES for messages in logged), logged diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 5f018ab5dda..5b31e0e5e3b 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -1236,3 +1236,58 @@ async def test_router_with_prompt_caching(anthropic_messages): print("response=", response) assert response._hidden_params["model_id"] == initial_model_id + + +class _LoggedMessagesProbe(CustomLogger): + def __init__(self) -> None: + self.logged_messages: list[object] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged_messages.append((kwargs["standard_logging_object"]["messages"], kwargs["messages"])) + + +@pytest.mark.asyncio +async def test_pin_is_written_with_message_logging_off_while_loggers_still_see_redacted_messages( + monkeypatch, local_model_cost_map +): + """ + `turn_off_message_logging` redacted the shared success-event payload before the check read it, so the + pin was keyed off `redacted-by-litellm`, never written, and a cacheable conversation bounced across the + group. The pin must land whatever the flag says, while a real logger alongside still sees only the + redacted messages. + """ + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + probe = _LoggedMessagesProbe() + monkeypatch.setattr(litellm, "callbacks", [probe]) + router = litellm.Router( + model_list=[ + { + "model_name": MODEL_GROUP_ALIAS, + "litellm_params": {"model": "anthropic/claude-opus-4-6", "api_key": "sk-fake"}, + "model_info": {"id": model_id}, + } + for model_id in ("dep-1", "dep-2") + ], + optional_pre_call_checks=["prompt_caching"], + ) + messages = _messages(word_count=5000) + + first = await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok") + served_by = first._hidden_params["model_id"] + + affinity_key = PromptCachingCache.get_prompt_caching_cache_key(messages, None) + pin = await _eventually(lambda: router.prompt_caching_pin_cache.get_cache(key=affinity_key)) + assert pin == {"model_id": served_by}, pin + + subsequent = [ + (await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok"))._hidden_params[ + "model_id" + ] + for _ in range(4) + ] + assert subsequent == [served_by] * 4 + + await _eventually(lambda: len(probe.logged_messages) == 5) + redacted = [{"role": "user", "content": "redacted-by-litellm"}] + assert probe.logged_messages == [(redacted, redacted)] * 5, probe.logged_messages From c7120f38acef209cd92e4f758b2c83290018ff61 Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 6 Nov 2026 03:04:59 +0000 Subject: [PATCH 08/18] build(deps): bump oauthlib to 4.0.1 in uv.lock Lock-only bump for GHSA-3jp2-3mrg-2vxc, which osv-scan flags on this line. uv lock --upgrade-package oauthlib, nothing else in the lock moves. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 674ffa049ef..ba0f16119ff 100644 --- a/uv.lock +++ b/uv.lock @@ -6194,11 +6194,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "4.0.0" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" }, ] [[package]] From 84a0d2950d949fddd3d07433aebb464d52128d39 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:30:41 -0800 Subject: [PATCH 09/18] fix(core): move drops_unsupported_params into core_helpers on lts/1.108.x Prerequisite of #48931 on lts/1.108.x, hand-ported from the core_helpers move in b51b9b7d43d (#47347), which the line does not carry. tool_fields imports the helper from core_helpers, and the responses bridge now imports it from there too. --- litellm/litellm_core_utils/core_helpers.py | 5 +++++ .../litellm_completion_transformation/transformation.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e706bcadef8..eae9f476863 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c import httpx from pydantic import TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason @@ -92,6 +93,10 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drops_unsupported_params(drop_params: object) -> bool: + return litellm.drop_params is True or normalize_drop_params(drop_params) is True + + def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: normalized: Final = normalize_drop_params(value) if normalized is None and value is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2e7fb6da194..31cd78c7517 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -41,7 +41,7 @@ from litellm._logging import verbose_logger from litellm.caching import InMemoryCache from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -322,10 +322,6 @@ def _dropped_cache_control(_: object) -> None: return None -def drops_unsupported_params(drop_params: object) -> bool: - return litellm.drop_params is True or normalize_drop_params(drop_params) is True - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item From 628f97a9e4336ce8b09de84fe41fc45edb020ebd Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:30:43 -0800 Subject: [PATCH 10/18] fix(mcp)!: answer 400 for a hosted MCP tool a translated route cannot run, drop it under drop_params (#48931) Backport of #48931 to lts/1.108.x. Cherry-picked from 61da6fcb4dc02cd58e39cb8c57ab3c99bea39a4f (main). --- .../prompt_templates/factory.py | 8 +- litellm/litellm_core_utils/tool_fields.py | 151 +++++++++++++++++- litellm/llms/anthropic/chat/transformation.py | 26 ++- .../pass_through/adapters/handler.py | 36 ++++- .../responses_adapters/handler.py | 17 +- .../bedrock/chat/converse_transformation.py | 16 +- .../bedrock/claude_platform/transformation.py | 3 + .../llms/databricks/chat/transformation.py | 26 +-- litellm/llms/deepseek/chat/transformation.py | 23 +-- litellm/llms/gemini/chat/transformation.py | 4 +- litellm/llms/gigachat/chat/transformation.py | 6 +- litellm/llms/ollama/chat/transformation.py | 13 +- .../vertex_and_google_ai_studio_gemini.py | 14 +- .../mcp_security/mcp_security_guardrail.py | 19 ++- litellm/types/llms/anthropic.py | 1 + litellm/utils.py | 31 +++- .../litellm_core_utils/test_tool_fields.py | 96 +++++++++++ .../test_anthropic_chat_transformation.py | 79 +++++++++ .../test_handler_output_config_passthrough.py | 116 +++++++++++++- .../test_responses_adapters_handler.py | 51 ++++++ .../chat/test_converse_transformation.py | 66 +++++++- .../test_databricks_chat_transformation.py | 68 ++++++++ .../chat/test_deepseek_chat_transformation.py | 45 ++++++ .../llms/gigachat/chat/test_transformation.py | 55 +++++++ .../ollama/test_ollama_chat_transformation.py | 67 ++++++++ .../test_ollama_completion_transformation.py | 70 ++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 84 ++++++++++ .../guardrail_hooks/test_mcp_security.py | 151 ++++++++++-------- .../test_litellm_completion_responses.py | 23 +++ .../llm_calls/responses_api.test.tsx | 44 ++++- .../components/llm_calls/responses_api.tsx | 55 ++----- 31 files changed, 1284 insertions(+), 180 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2613e0bbd28..d627edc3fe5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value from litellm.types.files import get_file_extension_from_mime_type @@ -5547,8 +5548,13 @@ def function_call_prompt( llm_provider: str, drop_params: bool, ) -> list[dict[str, object]]: + prompt_functions: Final = tools_without_hosted_mcp( + functions, model=model, route=llm_provider, drop_params=drop_params + ) + if not prompt_functions: + return messages header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:""" - function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions) + function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions) if not any(_is_system_message(message) for message in messages): return [*messages, {"role": "system", "content": function_prompt}] return [ diff --git a/litellm/litellm_core_utils/tool_fields.py b/litellm/litellm_core_utils/tool_fields.py index f90c8f5d174..5ee96ea9133 100644 --- a/litellm/litellm_core_utils/tool_fields.py +++ b/litellm/litellm_core_utils/tool_fields.py @@ -1,13 +1,16 @@ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence +from copy import copy from dataclasses import dataclass from functools import reduce -from typing import Annotated, Final, TypeAlias +from typing import Annotated, Final, TypeAlias, TypeVar from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params ROUTES_WITH_TOOLS: Final = frozenset( { @@ -162,3 +165,145 @@ def resolve_deployment_tool_fields( if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS: return kwargs return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),)) + + +_ToolT: Final = TypeVar("_ToolT") + +HOSTED_MCP_DROP_WARNING: Final = ( + "Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a " + "hosted MCP tool." +) + + +def _hosted_mcp_server_label(tool: object) -> str | None: + match tool: + case {"type": "mcp", "server_label": str() as server_label}: + return server_label + case {"type": "mcp"}: + return "unlabeled" + case _: + return None + + +def _drop_params_hint(dropped: str) -> str: + return ( + f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ) + + +def _one_log_line(text: str) -> str: + return text.replace("\r", "").replace("\n", "") + + +def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError: + return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400) + + +def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]: + server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None) + if not server_labels: + return tools + named_servers: Final = ", ".join(f"`{label}`" for label in server_labels) + if not drops_unsupported_params(drop_params): + raise _unrunnable_mcp_error( + f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM ' + "run the MCP server for this model, register it on the LiteLLM MCP gateway and send " + '`server_url: "litellm_proxy/mcp/"`. ' + _drop_params_hint("hosted MCP tools"), + model=model, + route=route, + ) + verbose_logger.warning( + HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route) + ) + return [tool for tool in tools if _hosted_mcp_server_label(tool) is None] + + +MCP_SERVERS_DROP_WARNING: Final = ( + "Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM " + "cannot pass mcp_servers to this route." +) + +ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset( + {"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"} +) + + +def _mcp_server_name(server: object) -> str: + match server: + case {"name": str() as name}: + return name + case _: + return "unnamed" + + +def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]: + if not mcp_servers: + return () + if not isinstance(mcp_servers, list): + return ("unnamed",) + return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers))) + + +def _is_mcp_toolset(tool: object) -> bool: + match tool: + case {"type": "mcp_toolset"}: + return True + case _: + return False + + +def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool: + additional_drop_params: Final = request_kwargs.get("additional_drop_params") + return drops_unsupported_params(request_kwargs.get("drop_params")) or ( + isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params + ) + + +def tools_without_mcp_servers( + mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool +) -> Sequence[_ToolT]: + server_names: Final = _mcp_server_names(mcp_servers) + if not server_names: + return tools + named_servers: Final = ", ".join(f"`{name}`" for name in server_names) + if not drops_unsupported_params(drop_params): + raise _unrunnable_mcp_error( + f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not " + 'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` ' + 'tool with `server_url: "litellm_proxy/mcp/"`. ' + _drop_params_hint("`mcp_servers`"), + model=model, + route=route, + ) + verbose_logger.warning( + MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route) + ) + return [tool for tool in tools if not _is_mcp_toolset(tool)] + + +_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"}) +_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object]) + + +def tool_params_without_hosted_mcp( + params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None +) -> _ParamsT: + tools: Final = params.get("tools") + if not isinstance(tools, list) or not tools: + return params + sent_tools: Final = _TOOLS.validate_python(tools) + if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or ( + runs_hosted_mcp is not None and runs_hosted_mcp() + ): + return params + kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params) + if len(kept_tools) == len(sent_tools): + return params + kept_params: Final = copy(params) + if kept_tools: + kept_params["tools"] = list(kept_tools) + return kept_params + dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS + for key in dropped_keys: + _ = kept_params.pop(key, None) + return kept_params diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c20241136b3..6092accc1de 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm +from litellm.anthropic_beta_headers_manager import get_unsupported_headers from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, @@ -35,6 +36,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, @@ -1205,6 +1207,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): *(tool for tool in tools[first_search_tool_index + 1 :] if not _is_openai_tool_search_tool(tool)), ] + def runs_hosted_mcp(self) -> bool: + return ANTHROPIC_BETA_HEADER_VALUES.MCP_CLIENT_2025_04_04.value not in get_unsupported_headers( + self._resolved_provider + ) + def _tools_param_with_openai_tool_search_mapped( self, non_default_params: Mapping[str, object], model: str, drop_params: bool ) -> Mapping[str, object]: @@ -1770,7 +1777,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model: str, drop_params: bool, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=non_default_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, + model=model, + route=self._resolved_provider, + drop_params=drop_params, + runs_hosted_mcp=self.runs_hosted_mcp, + ) + is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params) # NB: ``map_openai_params`` deliberately does NOT sanitize tool names # here. Names are the *original* OpenAI names at this stage, and must @@ -1783,7 +1797,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # via ``data = {**optional_params}`` and would 400 with # ``Extra inputs are not permitted``). - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value))) elif param == "tools": @@ -1797,8 +1811,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": _tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), + tool_choice=request_params.get("tool_choice"), + parallel_tool_use=request_params.get("parallel_tool_calls"), ) if _tool_choice is not None: @@ -1856,7 +1870,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # `thinking={type: enabled, budget_tokens}` interface a # pre-4.6 model actually supports instead of forwarding a # shape the model will reject. - max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens") + max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens") legacy_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort="medium", model=model, @@ -1949,7 +1963,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ## handle thinking tokens self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=optional_params + non_default_params=request_params, optional_params=optional_params ) return optional_params diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 7b8c2550838..8c0c065c457 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -14,6 +14,11 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.tool_fields import ( + ROUTES_WITHOUT_MCP_SERVERS, + drops_mcp_servers, + tools_without_mcp_servers, +) from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, ) @@ -588,6 +593,22 @@ class LiteLLMMessagesToCompletionTransformationHandler: Logging as LiteLLMLoggingObject, ) + provider_kwargs: Final = extra_kwargs or {} + custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider") + target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + strips_mcp_servers: Final = target_provider in ROUTES_WITHOUT_MCP_SERVERS + request_tools: Final = ( + tools_without_mcp_servers( + provider_kwargs.get("mcp_servers"), + tools or (), + model=model, + route=target_provider, + drop_params=drops_mcp_servers(provider_kwargs), + ) + if strips_mcp_servers + else tools + ) + request_data: Final[dict[str, object]] = { "model": model, "messages": messages, @@ -604,10 +625,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + if tool_choice and (request_tools or not tools): request_data["tool_choice"] = tool_choice - if tools: - request_data["tools"] = tools + if request_tools: + request_data["tools"] = request_tools if top_k is not None: request_data["top_k"] = top_k if top_p is not None: @@ -625,13 +646,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: if "output_config" in extra_kwargs: request_data["output_config"] = extra_kwargs["output_config"] - custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, - custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + custom_llm_provider=target_provider, ) if openai_request is None: @@ -659,7 +679,11 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Maintainability: when adding a new Anthropic-only request param to # ``AnthropicMessagesRequestOptionalParams``, also extend # ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak. - excluded_keys: Final = ANTHROPIC_ONLY_REQUEST_KEYS | CONVERTED_STREAM_KEYS | {"anthropic_messages"} + excluded_keys: Final = ( + ANTHROPIC_ONLY_REQUEST_KEYS + | CONVERTED_STREAM_KEYS + | ({"anthropic_messages", "mcp_servers"} if strips_mcp_servers else {"anthropic_messages"}) + ) # NOTE: extra_kwargs was already coerced from None to {} at the top of # this method (line ~220). It is guaranteed to be a dict here. for key, value in extra_kwargs.items(): diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 692e5d4c175..3eeaee8ad43 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( AllAnthropicMessageValues, @@ -141,6 +142,14 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + target_provider: Final = forwarded_kwargs.get("custom_llm_provider") + request_tools: Final = tools_without_mcp_servers( + forwarded_kwargs.get("mcp_servers"), + tools or (), + model=model, + route=target_provider if isinstance(target_provider, str) else "this route", + drop_params=drops_mcp_servers(forwarded_kwargs), + ) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ @@ -173,10 +182,10 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + if tool_choice and (request_tools or not tools): request_data["tool_choice"] = tool_choice - if tools: - request_data["tools"] = tools + if request_tools: + request_data["tools"] = list(request_tools) if top_p is not None: request_data["top_p"] = top_p if output_format: @@ -213,7 +222,7 @@ def _build_responses_kwargs( responses_kwargs["stream"] = True # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"} + excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"} for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6984fc1c1cd..943acfda2af 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -45,6 +45,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -1047,20 +1048,23 @@ class AmazonConverseConfig(BaseConfig): model: str, drop_params: bool, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route="bedrock", drop_params=drop_params + ) + is_thinking_enabled: Final = self.is_thinking_enabled(request_params) base_model: Final = BedrockModelInfo.get_base_model(model) drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( model=model, base_model=base_model ) - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "response_format" and isinstance(value, dict): optional_params = self._translate_response_format_param( value=value, model=model, optional_params=optional_params, - non_default_params=non_default_params, + non_default_params=request_params, is_thinking_enabled=is_thinking_enabled, ) if param == "max_tokens" or param == "max_completion_tokens": @@ -1113,7 +1117,7 @@ class AmazonConverseConfig(BaseConfig): ), ), model=model, - non_default_params=non_default_params, + non_default_params=request_params, optional_params=optional_params, ) if param == "tool_choice": @@ -1139,7 +1143,7 @@ class AmazonConverseConfig(BaseConfig): and value.get("type") == "adaptive" and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock") ): - max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens") + max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens") legacy_thinking = AnthropicConfig.map_reasoning_effort( reasoning_effort="medium", model=model, @@ -1216,7 +1220,7 @@ class AmazonConverseConfig(BaseConfig): # Nova 2 handles token budgeting differently through reasoningConfig if "gpt-oss" not in model and not self._is_nova_2_model(model): self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=optional_params + non_default_params=request_params, optional_params=optional_params ) final_is_thinking_enabled: Final = self.is_thinking_enabled(optional_params) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index cea1140bf65..047f9d342e6 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -29,6 +29,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + def runs_hosted_mcp(self) -> bool: + return True + def validate_environment( self, headers: dict, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2145c19a71a..8c96cc4f185 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_internal_message_fields, strip_name_from_message, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict @@ -283,7 +284,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): complete_url: Final = f"{url_base}/chat/completions" return complete_url - def get_supported_openai_params(self, model: str | None = None) -> list: + def get_supported_openai_params(self, model: str | None = None) -> list[str]: return [ "stream", "stop", @@ -389,8 +390,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): drop_params: bool, replace_max_completion_tokens_with_max_tokens: bool = True, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) - mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) + request_params: Final = ( + tool_params_without_hosted_mcp(non_default_params, model=model, route="databricks", drop_params=drop_params) + if "claude" in model + else non_default_params + ) + is_thinking_enabled: Final = self.is_thinking_enabled(request_params) + mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params) if "claude" in model: AnthropicConfig.translate_legacy_thinking_for_adaptive_model( model=model, optional_params=mapped_params, custom_llm_provider="databricks", drop_params=drop_params @@ -399,16 +405,16 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_params["tools"] = self._map_openai_to_dbrx_tool( model=model, tools=mapped_params["tools"], drop_params=drop_params ) - if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens: - mapped_params["max_tokens"] = non_default_params[ + if "max_completion_tokens" in request_params and replace_max_completion_tokens_with_max_tokens: + mapped_params["max_tokens"] = request_params[ "max_completion_tokens" ] # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens' mapped_params.pop("max_completion_tokens", None) - if "response_format" in non_default_params and "claude" in model: + if "response_format" in request_params and "claude" in model: _tool: Final = self.map_response_format_to_databricks_tool( model, - non_default_params["response_format"], + request_params["response_format"], mapped_params, is_thinking_enabled, ) @@ -426,8 +432,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): "response_format", None ) # unsupported for claude models - if json_schema -> convert to tool call - if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model): - reasoning_effort_value: Final = non_default_params.get("reasoning_effort") + if "reasoning_effort" in request_params and self._uses_anthropic_thinking_param(model): + reasoning_effort_value: Final = request_params.get("reasoning_effort") mapped_thinking: Final = AnthropicConfig._map_reasoning_effort( reasoning_effort=reasoning_effort_value, model=model, @@ -453,7 +459,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): optional_params.pop("reasoning_effort", None) ## handle thinking tokens self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=mapped_params + non_default_params=request_params, optional_params=mapped_params ) return mapped_params diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index d3319856d29..e1e9694e2ab 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import supports_reasoning, supports_vision @@ -18,17 +19,15 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class DeepSeekChatConfig(OpenAIGPTConfig): - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list[str]: """ DeepSeek reasoner models support thinking parameter. """ - params: Final = super().get_supported_openai_params(model) - params.extend(["thinking", "reasoning_effort"]) - return params + return [*super().get_supported_openai_params(model), "thinking", "reasoning_effort"] def map_openai_params( self, - non_default_params: dict, + non_default_params: dict[str, object], optional_params: dict, model: str, drop_params: bool, @@ -44,7 +43,12 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Reference: https://api-docs.deepseek.com/guides/thinking_mode """ # Let parent handle standard params first - optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + optional_params = super().map_openai_params( + tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params), + optional_params, + model, + drop_params, + ) # Pop thinking/reasoning_effort from optional_params first (parent may have added them) # Then re-add only if valid for DeepSeek @@ -242,6 +246,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): tools: Final = optional_params.get("tools") if not isinstance(tools, list) or not tools: return optional_params + sent_tools: Final[tuple[object, ...]] = tuple(tools) def _is_function_tool(tool: object) -> bool: return isinstance(tool, dict) and tool.get("type") == "function" @@ -266,14 +271,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig): name: Final = function.get("name") return isinstance(name, str) and name in function_tool_names - function_tools: Final = [tool for tool in tools if _is_function_tool(tool)] - if len(function_tools) == len(tools): + function_tools: Final = [tool for tool in sent_tools if _is_function_tool(tool)] + if len(function_tools) == len(sent_tools): return optional_params dropped_types: Final = sorted( { str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__ - for tool in tools + for tool in sent_tools if not _is_function_tool(tool) } ) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 143652ee77c..79842017c34 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Final, cast +from typing import ClassVar, Final, cast import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -49,6 +49,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): Note: Please make sure to modify the default parameters as required for your use case. """ + _route: ClassVar[str] = "gemini" + temperature: float | None = None max_output_tokens: int | None = None top_p: float | None = None diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 1fe7de0ba85..3ec46cb7e88 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -17,6 +17,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.gigachat.utils import convert_usage, get_api_base from litellm.secret_managers.main import get_secret_str @@ -160,7 +161,10 @@ class GigaChatConfig(BaseConfig): drop_params: bool, ) -> dict: # mutable-ok: base class contract returns dict """Map OpenAI parameters to GigaChat parameters.""" - for param, value in non_default_params.items(): + request_params: Final = tool_params_without_hosted_mcp( + dict(non_default_params), model=model, route="gigachat", drop_params=drop_params + ) + for param, value in request_params.items(): if param == "stream": optional_params["stream"] = value elif param == "temperature": diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 644ab187f15..46b4caad43f 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, cast from httpx._models import Headers, Response -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._uuid import uuid @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.ollama import ( @@ -38,6 +39,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) + class OllamaChatConfig(BaseConfig): """ @@ -150,7 +153,13 @@ class OllamaChatConfig(BaseConfig): model: str, drop_params: bool, ) -> dict: - for param, value in non_default_params.items(): + request_params: Final = tool_params_without_hosted_mcp( + _REQUEST_PARAMS.validate_python(non_default_params), + model=model, + route="ollama_chat", + drop_params=drop_params, + ) + for param, value in request_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["num_predict"] = value if param == "stream": diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 5faf24ac92b..311d23f363d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -8,7 +8,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args import httpx from pydantic import JsonValue @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -279,6 +280,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Note: Please make sure to modify the default parameters as required for your use case. """ + _route: ClassVar[str] = "vertex_ai" + temperature: float | None = None max_output_tokens: int | None = None top_p: float | None = None @@ -1245,9 +1248,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model: str, drop_params: bool, ) -> dict: - self._apply_include_server_side_tool_invocations(non_default_params, optional_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route=self._route, drop_params=drop_params + ) + self._apply_include_server_side_tool_invocations(request_params, optional_params) gemini_sampling_params_warned: bool = False - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): if value is not None and value < 1.0: @@ -1317,7 +1323,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value elif param == "parallel_tool_calls": - tools_list = non_default_params.get("tools", non_default_params.get("functions")) + tools_list = request_params.get("tools", request_params.get("functions")) num_tools = len(tools_list) if isinstance(tools_list, list) else 0 # Gemini does not support parallel_tool_calls=False with multiple # tools. Drop the param instead of failing — Responses API clients diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 8482567b6da..9cd5555fe25 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -104,9 +104,20 @@ class MCPSecurityGuardrail(CustomGuardrail): from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy.proxy_server import prisma_client async with global_mcp_server_manager.catalog.operation(): - registry: Final = global_mcp_server_manager.get_registry() - registered_names: Final = set(registry.keys()) - - return frozenset(requested_servers - registered_names) + not_servers: Final = frozenset( + name + for name in requested_servers + if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None + ) + if prisma_client is None: + return not_servers + return frozenset( + [ + name + for name in not_servers + if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None + ] + ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f113f833a46..53abac75162 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -807,6 +807,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01" PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01" DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03" + MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..fea6b9d2177 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -93,7 +93,11 @@ from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens -from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields +from litellm.litellm_core_utils.tool_fields import ( + MalformedField, + resolve_deployment_tool_fields, + tools_without_hosted_mcp, +) from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.catalog import decision from litellm.rust_bridge.configuration import Decision @@ -4690,7 +4694,12 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict: return d -def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict: +_TOOL_LIST: Final = TypeAdapter(list[object]) + + +def pre_process_optional_params( + passed_params: dict, non_default_params: dict, custom_llm_provider: str, model: str, drop_params: bool | None +) -> dict: """For .completion(), preprocess optional params""" optional_params: dict = {} @@ -4747,14 +4756,20 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c and custom_llm_provider not in litellm.openai_compatible_providers ): if custom_llm_provider == "ollama": + if "tools" in non_default_params: + sent_tools: Final = _TOOL_LIST.validate_python(non_default_params.pop("tools")) + non_default_params.pop("tool_choice", None) # causes ollama requests to hang + prompt_tools: Final = tools_without_hosted_mcp( + sent_tools, model=model, route=custom_llm_provider, drop_params=bool(drop_params) + ) + if sent_tools and not prompt_tools: + return optional_params + optional_params["functions_unsupported_model"] = prompt_tools + elif "functions" in non_default_params: + optional_params["functions_unsupported_model"] = non_default_params.pop("functions") # ollama actually supports json output optional_params["format"] = "json" litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt - if "tools" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("tools") - non_default_params.pop("tool_choice", None) # causes ollama requests to hang - elif "functions" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("functions") elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead optional_params["functions_unsupported_model"] = non_default_params.pop( "tools", non_default_params.pop("functions", None) @@ -4841,6 +4856,8 @@ def get_optional_params( passed_params=passed_params, non_default_params=non_default_params, custom_llm_provider=custom_llm_provider, + model=model, + drop_params=drop_params, ) def _check_valid_arg(supported_params: list[str]): diff --git a/tests/unit/litellm_core_utils/test_tool_fields.py b/tests/unit/litellm_core_utils/test_tool_fields.py index 23d7de98fbc..a99278d18dd 100644 --- a/tests/unit/litellm_core_utils/test_tool_fields.py +++ b/tests/unit/litellm_core_utils/test_tool_fields.py @@ -1,4 +1,5 @@ import datetime +import logging from collections.abc import Callable, Mapping from typing import Final @@ -6,7 +7,13 @@ import pytest import litellm from litellm import Router +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.tool_fields import ( + tool_params_without_hosted_mcp, + tools_without_hosted_mcp, + tools_without_mcp_servers, +) from litellm.utils import async_pre_call_deployment_hook _GOOD_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} @@ -185,3 +192,92 @@ async def test_tools_core_accepts_reach_the_deployment(tools: object) -> None: sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion") assert sent["tools"] is tools + + +def test_a_hosted_mcp_error_names_every_server_and_marks_an_unlabeled_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + tools_without_hosted_mcp( + tools=[ + {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}, + {"type": "mcp", "server_url": "https://mcp.example.com/mcp"}, + ], + model="gemini-3.8-flash", + route="gemini", + drop_params=False, + ) + assert "the hosted MCP tool `deepwiki`, `unlabeled` " in str(raised.value) + + +_HOSTED_MCP_TOOL: Final = {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"} + + +def test_dropping_a_hosted_mcp_tool_keeps_the_other_tool_params_and_leaves_the_caller_dict_alone() -> None: + tool_choice: Final = {"type": "function", "function": {"name": "get_weather"}} + sent: Final = { + "tools": [_HOSTED_MCP_TOOL, _GOOD_TOOL], + "tool_choice": tool_choice, + "parallel_tool_calls": False, + "temperature": 0.2, + } + params: Final = tool_params_without_hosted_mcp(sent, model="deepseek-v4-flash", route="deepseek", drop_params=True) + + assert params == { + "tools": [_GOOD_TOOL], + "tool_choice": tool_choice, + "parallel_tool_calls": False, + "temperature": 0.2, + } + assert sent["tools"] == [_HOSTED_MCP_TOOL, _GOOD_TOOL] + + +def test_dropping_every_hosted_mcp_tool_clears_the_tool_selection_params() -> None: + params: Final = tool_params_without_hosted_mcp( + {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required", "parallel_tool_calls": True, "temperature": 0.2}, + model="deepseek-v4-flash", + route="deepseek", + drop_params=True, + ) + + assert params == {"temperature": 0.2} + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self.messages: Final[list[str]] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.mark.parametrize( + "drop", + [ + lambda model: tools_without_hosted_mcp( + [{"type": "mcp", "server_label": "wiki\r\nFORGED", "server_url": "https://mcp.example.com/mcp"}], + model=model, + route="deepseek", + drop_params=True, + ), + lambda model: tools_without_mcp_servers( + [{"type": "url", "name": "wiki\r\nFORGED", "url": "https://mcp.example.com/mcp"}], + tools=[], + model=model, + route="bedrock", + drop_params=True, + ), + ], + ids=["hosted_mcp_tool", "mcp_servers"], +) +def test_a_drop_warning_stays_on_one_log_line(drop: Callable[[str], object]) -> None: + records: Final = _Records() + verbose_logger.addHandler(records) + try: + drop("deepseek-v4-flash\nFORGED level=CRITICAL") + finally: + verbose_logger.removeHandler(records) + + assert len(records.messages) == 1 + assert "\n" not in records.messages[0] and "\r" not in records.messages[0] + assert "FORGED level=CRITICAL" in records.messages[0] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 9e70d213f47..8dff6864e8f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9641,3 +9641,82 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} +_ROUTES_WITHOUT_THE_MCP_CONNECTOR: Final = ( + pytest.param(AmazonAnthropicClaudeConfig, "bedrock", id="bedrock_invoke"), + pytest.param(VertexAIAnthropicConfig, "vertex_ai", id="vertex_ai"), +) + + +@pytest.mark.parametrize("config_class", [AnthropicConfig, AzureAnthropicConfig, BedrockClaudePlatformConfig]) +def test_routes_with_the_mcp_connector_send_a_hosted_mcp_tool_as_an_mcp_server(monkeypatch, config_class): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=False, + ) + assert [server["url"] for server in mapped["mcp_servers"]] == [_HOSTED_MCP_TOOL["server_url"]] + assert [tool["name"] for tool in mapped["tools"]] == ["get_weather"] + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_a_hosted_mcp_tool_answers_400_on_claude_routes_without_the_mcp_connector(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert f"claude-haiku-5-5 on {route} " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_claude_routes_without_the_mcp_connector_drop_a_hosted_mcp_tool_under_drop_params( + monkeypatch, caplog, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=True, + ) + assert [tool["name"] for tool in mapped["tools"]] == ["get_weather"] + assert "mcp_servers" not in mapped + assert any( + "`deepwiki`" in record.getMessage() and f" on {route} (" in record.getMessage() for record in caplog.records + ) + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_claude_routes_without_the_mcp_connector_send_no_tools_when_drop_params_drops_every_one( + monkeypatch, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=True, + ) + assert "tools" not in mapped + assert "mcp_servers" not in mapped + assert "tool_choice" not in mapped diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py index 03553e65ad4..55287fa070d 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py @@ -27,11 +27,14 @@ Tests cover (consolidating PRs #23706 and #22727): import os import sys +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch import pytest +import litellm + # Anchor sys.path to this file's location — not the working-directory-relative # pattern Greptile flagged on PR #23706. Resolves correctly regardless of # where pytest is invoked from. @@ -67,8 +70,8 @@ def _call_prepare(extra_kwargs, model="gpt-4o", output_format=None, **overrides) system=None, temperature=None, thinking=None, - tool_choice=None, - tools=None, + tool_choice=overrides.get("tool_choice"), + tools=overrides.get("tools"), top_k=None, top_p=None, output_format=output_format, @@ -263,3 +266,112 @@ async def test_messages_layer_converted_stream_flag_still_gets_a_message_back(co ) assert response["content"][0]["text"] == "hi" + + +_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"} + + +@pytest.mark.parametrize( + ("mcp_servers", "named"), + [ + ([_DEEPWIKI_SERVER], "`deepwiki`"), + ([_DEEPWIKI_SERVER, {"type": "url", "url": "https://mcp.example.com/mcp"}], "`deepwiki`, `unnamed`"), + ({"name": "deepwiki"}, "`unnamed`"), + ], + ids=["named", "one-unnamed", "not-a-list"], +) +def test_mcp_servers_a_chat_route_cannot_run_answer_400_naming_each_server( + monkeypatch: pytest.MonkeyPatch, mcp_servers: object, named: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + _call_prepare( + extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash" + ) + + assert raised.value.status_code == 400 + assert f"to gemini-3.8-flash on gemini, so the MCP server {named} would not run" in str(raised.value) + + +@pytest.mark.parametrize( + ("route", "forwards"), + [("litellm_proxy", True), ("hosted_vllm", True), ("bedrock", False), ("vertex_ai", False)], +) +def test_mcp_servers_reach_routes_that_carry_them_and_answer_400_elsewhere( + monkeypatch: pytest.MonkeyPatch, route: str, forwards: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + extra_kwargs: Final = {"mcp_servers": [_DEEPWIKI_SERVER], "custom_llm_provider": route} + if not forwards: + with pytest.raises(litellm.UnsupportedParamsError, match=f"on {route}, so the MCP server `deepwiki`"): + _call_prepare(extra_kwargs=extra_kwargs, model="some-model") + return + + completion_kwargs, _ = _call_prepare(extra_kwargs=extra_kwargs, model="some-model") + + assert completion_kwargs["mcp_servers"] == [_DEEPWIKI_SERVER] + + +_DEEPWIKI_TOOLSET: Final = {"type": "mcp_toolset", "mcp_server_name": "deepwiki"} +_WEATHER_TOOL: Final = { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, +} + + +@pytest.mark.parametrize( + ("tools", "kept_tool_names"), + [([_DEEPWIKI_TOOLSET, _WEATHER_TOOL], ["get_weather"]), ([_DEEPWIKI_TOOLSET], None)], + ids=["with-function", "toolset-only"], +) +def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers( + monkeypatch: pytest.MonkeyPatch, tools: list[dict[str, object]], kept_tool_names: list[str] | None +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + completion_kwargs, _ = _call_prepare( + extra_kwargs={"mcp_servers": [_DEEPWIKI_SERVER], "custom_llm_provider": "gemini"}, + model="gemini-3.8-flash", + tools=tools, + tool_choice={"type": "auto"}, + ) + + sent_tools: Final = completion_kwargs.get("tools") + sent_names: Final = [tool["function"]["name"] for tool in sent_tools] if sent_tools else None + assert sent_names == kept_tool_names + assert ("tool_choice" in completion_kwargs) == bool(kept_tool_names) + + +_DROP_FORMS: Final[Mapping[str, Mapping[str, object]]] = { + "request": {"drop_params": True}, + "global": {}, + "additional_drop_params": {"additional_drop_params": ["mcp_servers"]}, +} + + +@pytest.mark.parametrize("drop_params", list(_DROP_FORMS)) +def test_drop_params_drops_mcp_servers_instead_of_forwarding_them( + monkeypatch: pytest.MonkeyPatch, drop_params: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params == "global") + completion_kwargs, _ = _call_prepare( + extra_kwargs={ + "mcp_servers": [_DEEPWIKI_SERVER], + "custom_llm_provider": "gemini", + **_DROP_FORMS[drop_params], + }, + model="gemini-3.8-flash", + ) + + assert "mcp_servers" not in completion_kwargs + assert completion_kwargs["messages"] == MESSAGES + + +@pytest.mark.parametrize("mcp_servers", [None, []], ids=["absent", "empty"]) +def test_no_mcp_servers_needs_no_drop_params(monkeypatch: pytest.MonkeyPatch, mcp_servers: object) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + completion_kwargs, _ = _call_prepare( + extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash" + ) + + assert "mcp_servers" not in completion_kwargs diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 453429df084..6aec78bdb97 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -633,3 +633,54 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"} + + +def test_mcp_servers_the_responses_bridge_cannot_run_answer_400(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER]}, + max_tokens=1024, + messages=MESSAGES, + ) + + assert rejected.value.status_code == 400 + assert "to gpt-5.6 on openai, so the MCP server `deepwiki` would not run" in rejected.value.message + + +@pytest.mark.parametrize( + "drop_form", + [{"drop_params": True}, {"additional_drop_params": ["mcp_servers"]}], + ids=["drop_params", "additional_drop_params"], +) +def test_drop_params_keeps_mcp_servers_off_the_responses_request( + monkeypatch: pytest.MonkeyPatch, drop_form: Mapping[str, object] +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + responses_kwargs: Final = _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER], **drop_form}, + max_tokens=1024, + messages=MESSAGES, + ) + + assert "mcp_servers" not in responses_kwargs + assert responses_kwargs["input"] + + +def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + weather_tool: Final = {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}} + responses_kwargs: Final = _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER], "drop_params": True}, + max_tokens=1024, + messages=MESSAGES, + tools=[{"type": "mcp_toolset", "mcp_server_name": "deepwiki"}, weather_tool], + ) + + assert [tool["name"] for tool in responses_kwargs["tools"]] == ["get_weather"] diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6c996a7ce75..8c1815a779a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -3,7 +3,7 @@ import copy import json import os import re -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -10147,3 +10147,67 @@ def test_map_openai_params_drops_the_openai_tool_search_tool_and_every_deferral_ ) assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather", "get_stock_price"] assert all("defer_loading" not in tool and "defer_loading" not in tool["function"] for tool in mapped["tools"]) + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def _converse_request_for_tools( + model: str, non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object]: + config: Final = AmazonConverseConfig() + return config._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params=config.map_openai_params( + non_default_params=dict(non_default_params), optional_params={}, model=model, drop_params=drop_params + ), + messages=[{"role": "user", "content": "hi"}], + ) + + +@pytest.mark.parametrize( + "model", ["us.openai.gpt-oss-120b-1:0", "us.anthropic.claude-haiku-5-5", "us.amazon.nova-2-lite-v1:0"] +) +def test_a_hosted_mcp_tool_answers_400_on_converse_without_drop_params(monkeypatch, model): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + _converse_request_for_tools(model, {"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, drop_params=False) + assert raised.value.status_code == 400 + assert model in str(raised.value) + assert "`deepwiki`" in str(raised.value) + assert 'server_url: "litellm_proxy/mcp/"' in str(raised.value) + assert "drop_params: true" in str(raised.value) + + +def test_converse_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + model: Final = "us.openai.gpt-oss-120b-1:0" + with caplog.at_level("WARNING"): + request: Final = _converse_request_for_tools( + model, {"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, drop_params=True + ) + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and f"model={model} on bedrock (" in record.getMessage() + for record in caplog.records + ) + + +def test_converse_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _converse_request_for_tools( + "us.openai.gpt-oss-120b-1:0", {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "auto"}, drop_params=True + ) + assert "toolConfig" not in request + assert "toolChoice" not in json.dumps(request) + assert "tool_choice" not in json.dumps(request) diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 650517c92ca..d56bdeefda6 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2447,3 +2447,71 @@ def test_databricks_claude_drops_the_openai_tool_search_tool_under_drop_params(m assert mapped["tools"] == [ {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}} ] + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def test_databricks_claude_answers_400_for_a_hosted_mcp_tool_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + DatabricksConfig().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert "databricks-claude-opus-5-5 on databricks " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_databricks_claude_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and " on databricks (" in record.getMessage() for record in caplog.records + ) + + +def test_databricks_claude_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert "tools" not in mapped + assert "tool_choice" not in mapped + + +def test_databricks_claude_still_sends_max_completion_tokens_as_max_tokens_when_dropping_a_hosted_mcp_tool( + monkeypatch, +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL], "max_completion_tokens": 64}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert mapped["max_tokens"] == 64 + assert "max_completion_tokens" not in mapped + assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather"] diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index dd99ad8bf91..ab38f4fc41f 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -907,3 +907,48 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "https://gateway.example.com/deepseek/chat/completions" ) + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} + + +def test_a_hosted_mcp_tool_answers_400_on_deepseek_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + DeepSeekChatConfig().map_openai_params( + {"tools": [_function_tool("shell"), _HOSTED_MCP_TOOL]}, {}, "deepseek-v4-flash", False + ) + assert raised.value.status_code == 400 + assert "deepseek-v4-flash on deepseek " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_deepseek_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL, _function_tool("shell")], "tool_choice": "auto"}, {}, "deepseek-v4-flash", True + ) + assert result["tools"] == [_function_tool("shell")] + assert result["tool_choice"] == "auto" + assert any( + "`deepwiki`" in record.getMessage() and " on deepseek (" in record.getMessage() for record in caplog.records + ) + + +def test_deepseek_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required", "parallel_tool_calls": True}, + {}, + "deepseek-v4-flash", + True, + ) + assert "tools" not in result + assert "tool_choice" not in result + assert "parallel_tool_calls" not in result diff --git a/tests/unit/llms/gigachat/chat/test_transformation.py b/tests/unit/llms/gigachat/chat/test_transformation.py index 9116a4d1979..4d8a916ea87 100644 --- a/tests/unit/llms/gigachat/chat/test_transformation.py +++ b/tests/unit/llms/gigachat/chat/test_transformation.py @@ -7,10 +7,12 @@ Run with: pytest tests/unit/llms/gigachat/chat/test_transformation.py -v import asyncio import importlib +from typing import Final import pytest import litellm +from litellm.llms.gigachat.chat.transformation import GigaChatConfig from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -575,3 +577,56 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), "cohere_key": getattr(litellm, "cohere_key", None), } + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def test_a_hosted_mcp_tool_answers_400_on_gigachat_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + GigaChatConfig().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert "GigaChat-2-Max on gigachat " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_gigachat_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = GigaChatConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=True, + ) + assert [function["name"] for function in mapped["functions"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and " on gigachat (" in record.getMessage() for record in caplog.records + ) + + +def test_gigachat_sends_no_functions_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = GigaChatConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "auto"}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=True, + ) + assert "functions" not in mapped + assert "function_call" not in mapped diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 1a0bbe171a9..ae75c40d32a 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -5,6 +5,7 @@ import sys from typing import Final, cast import pytest +import respx from pydantic import BaseModel sys.path.insert( @@ -1108,3 +1109,69 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/chat" ) + + +_DEEPWIKI_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, +} + + +def _ollama_chat_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("http://ollama.test:11434/api/chat").respond( + json={ + "model": "qwen3:0.6b", + "message": {"role": "assistant", "content": "Sunny"}, + "done": True, + "prompt_eval_count": 5, + "eval_count": 3, + } + ) + + +def test_ollama_chat_answers_400_for_a_hosted_mcp_tool_it_cannot_run( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = _ollama_chat_route(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.completion( + model="ollama_chat/qwen3:0.6b", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], + api_base="http://ollama.test:11434", + ) + + assert raised.value.status_code == 400 + assert "qwen3:0.6b on ollama_chat cannot run the hosted MCP tool `deepwiki`" in str(raised.value) + assert not route.called + + +@pytest.mark.parametrize( + ("tools", "sent_tools"), + [([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], [_WEATHER_TOOL]), ([_DEEPWIKI_MCP_TOOL], None)], + ids=["beside-a-function", "alone"], +) +def test_ollama_chat_drop_params_sends_only_the_tools_ollama_can_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], sent_tools: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = _ollama_chat_route(respx_mock) + + response: Final = litellm.completion( + model="ollama_chat/qwen3:0.6b", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=tools, + api_base="http://ollama.test:11434", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content).get("tools") == sent_tools + assert response.choices[0].message.content == "Sunny" diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 248e745ebff..0cf18ff6e0d 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -990,3 +990,73 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/generate" ) + + +_DEEPWIKI_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} + + +def _ollama_generate_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("http://ollama.test:11434/api/generate").respond( + json={"model": "llama3.2", "response": "Sunny", "done": True, "prompt_eval_count": 5, "eval_count": 3} + ) + + +def test_ollama_answers_400_for_a_hosted_mcp_tool_instead_of_pasting_it_into_the_prompt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "add_function_to_prompt", False) + route: Final = _ollama_generate_route(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.completion( + model="ollama/llama3.2", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], + api_base="http://ollama.test:11434", + ) + + assert raised.value.status_code == 400 + assert "cannot run the hosted MCP tool `deepwiki`" in str(raised.value) + assert not route.called + + +@pytest.mark.parametrize( + ("tools", "prompt_names_weather"), + [([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], True), ([_DEEPWIKI_MCP_TOOL], False)], + ids=["beside-a-function", "alone"], +) +def test_ollama_drop_params_keeps_a_hosted_mcp_tool_out_of_the_prompt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], prompt_names_weather: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "add_function_to_prompt", False) + route: Final = _ollama_generate_route(respx_mock) + + litellm.completion( + model="ollama/llama3.2", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=tools, + api_base="http://ollama.test:11434", + drop_params=True, + ) + + body: Final = json.loads(route.calls.last.request.content) + prompt: Final = body["prompt"] + assert (body.get("format") == "json") is prompt_names_weather + assert "deepwiki" not in prompt + assert ("get_weather" in prompt) is prompt_names_weather + assert prompt.startswith("### User:\nweather in Paris?") + assert prompt_names_weather or prompt == "### User:\nweather in Paris?\n\n" diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3d7296ffa7b..e014b620dec 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -9887,3 +9887,87 @@ async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_n sent: Final = await _sent_thinking_config("gemini-3.8-flash", {"reasoning_effort": "high"}) assert sent["includeThoughts"] is expected + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} +_GEMINI_ROUTES: Final = ( + pytest.param(VertexGeminiConfig, "vertex_ai", id="vertex_ai"), + pytest.param(GoogleAIStudioGeminiConfig, "gemini", id="gemini"), +) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_a_hosted_mcp_tool_answers_400_on_gemini_without_drop_params(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert f"gemini-3.8-flash on {route} " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + assert "litellm_proxy/mcp/" in str(raised.value) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool( + monkeypatch, caplog, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "deepwiki" not in json.dumps(mapped) + assert "get_weather" in json.dumps(mapped["tools"]) + assert any( + "`deepwiki`" in record.getMessage() and f" on {route} (" in record.getMessage() for record in caplog.records + ) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "tools" not in mapped + assert "tool_choice" not in mapped + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_keeps_tool_choice_none_for_legacy_functions_after_dropping_every_hosted_mcp_tool( + monkeypatch, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={ + "tools": [_HOSTED_MCP_TOOL], + "functions": [_WEATHER_FUNCTION_TOOL["function"]], + "tool_choice": "none", + }, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "deepwiki" not in json.dumps(mapped) + assert "get_weather" in json.dumps(mapped["tools"]) + assert mapped["tool_choice"] == {"functionCallingConfig": {"mode": "NONE"}} diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py index e4c7bcba951..6501526634c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py @@ -6,19 +6,24 @@ and allows requests with only registered servers. Covers both /chat/completions and /responses API paths (same pre_call_hook logic, different call_type). """ -from typing import Literal -from unittest.mock import MagicMock, patch +from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException import litellm +from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.mcp_security import initialize_guardrail from litellm.proxy.guardrails.guardrail_hooks.mcp_security.mcp_security_guardrail import ( MCPSecurityGuardrail, ) +from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams +from litellm.types.mcp import MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.types.mcp_server.mcp_toolset import MCPToolset @pytest.fixture @@ -60,93 +65,73 @@ class TestExtractMCPServerNames: assert names == set() +def _manager_with_saved_server() -> MCPServerManager: + manager: Final = MCPServerManager() + server: Final = MCPServer( + server_id="3f6b2c1e-saved-zapier", + name="zapier", + server_name="zapier", + alias="zap", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + ) + manager.registry[server.server_id] = server + return manager + + class TestMCPSecurityGuardrailPreCall: @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_blocks_unregistered_server_chat_completions( - self, mock_manager, guardrail - ): - """Simulates /chat/completions with an unregistered MCP server.""" - mock_manager.get_registry.return_value = {"zapier": MagicMock()} - + @pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) + async def test_blocks_unregistered_server(self, guardrail, call_type): data = { "tools": [ {"type": "mcp", "server_url": "litellm_proxy/mcp/zapier"}, {"type": "mcp", "server_url": "litellm_proxy/mcp/evil_server"}, ], "model": "gpt-4", - "messages": [{"role": "user", "content": "hi"}], "guardrails": ["test-mcp-security"], } - with pytest.raises(HTTPException) as exc_info: + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _manager_with_saved_server(), + ), + pytest.raises(HTTPException) as exc_info, + ): await guardrail.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), cache=MagicMock(), data=data, - call_type="acompletion", + call_type=call_type, ) assert exc_info.value.status_code == 400 - assert "evil_server" in str(exc_info.value.detail) + assert exc_info.value.detail["unregistered_servers"] == ["evil_server"] @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_blocks_unregistered_server_responses_api( - self, mock_manager, guardrail - ): - """Simulates /responses with an unregistered MCP server.""" - mock_manager.get_registry.return_value = {"github": MagicMock()} - + @pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) + @pytest.mark.parametrize("reference", ["zapier", "zap", "3f6b2c1e-saved-zapier"]) + async def test_allows_saved_server_by_any_name_the_gateway_routes(self, guardrail, call_type, reference): data = { - "tools": [ - {"type": "mcp", "server_url": "litellm_proxy/mcp/unknown_server"}, - ], - "model": "gpt-4o", - "input": "What can you do?", - "guardrails": ["test-mcp-security"], - } - - with pytest.raises(HTTPException) as exc_info: - await guardrail.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth(), - cache=MagicMock(), - data=data, - call_type="aresponses", - ) - assert exc_info.value.status_code == 400 - assert "unknown_server" in str(exc_info.value.detail) - - @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_allows_registered_servers(self, mock_manager, guardrail): - """All MCP servers are registered - request passes through.""" - mock_manager.get_registry.return_value = { - "zapier": MagicMock(), - "github": MagicMock(), - } - - data = { - "tools": [ - {"type": "mcp", "server_url": "litellm_proxy/mcp/zapier"}, - {"type": "mcp", "server_url": "litellm_proxy/mcp/github"}, - ], + "tools": [{"type": "mcp", "server_url": f"litellm_proxy/mcp/{reference}"}], "model": "gpt-4", - "messages": [{"role": "user", "content": "hi"}], "guardrails": ["test-mcp-security"], } - result = await guardrail.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth(), - cache=MagicMock(), - data=data, - call_type="acompletion", - ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _manager_with_saved_server(), + ), + ): + result = await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(), + data=data, + call_type=call_type, + ) assert result == data @pytest.mark.asyncio @@ -221,11 +206,13 @@ async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_ prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(side_effect=([row], [])) prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(return_value=None) data = {"tools": [{"type": "mcp", "server_url": "litellm_proxy/mcp/peer-server"}], "guardrails": ["test-mcp-security"]} with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), ): result = await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) @@ -234,3 +221,37 @@ async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_ await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) assert exc.value.status_code == 400 assert exc.value.detail["unregistered_servers"] == ["peer-server"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) +async def test_allows_saved_toolset_by_name_and_blocks_unknown_name(guardrail, call_type): + saved_toolset: Final = MCPToolset(toolset_id="7c1d-research", toolset_name="research") + + async def find_toolset(*, where: dict[str, str]) -> MCPToolset | None: + return saved_toolset if where["toolset_name"] == saved_toolset.toolset_name else None + + prisma: Final = MagicMock() + prisma.writer_db = prisma.db + prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(side_effect=find_toolset) + data: Final = { + "tools": [ + {"type": "mcp", "server_url": "litellm_proxy/mcp/research"}, + {"type": "mcp", "server_url": "litellm_proxy/mcp/evil_server"}, + ], + "guardrails": ["test-mcp-security"], + } + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + MCPServerManager(), + ), + pytest.raises(HTTPException) as exc, + ): + await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) + assert exc.value.detail["unregistered_servers"] == ["evil_server"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 79d5ec5d877..4fb2a4af571 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6536,3 +6536,26 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun assert incomplete_event.response.usage is not None assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text) assert incomplete_event.response.usage.input_tokens > 0 + + +def test_a_hosted_mcp_tool_sent_to_responses_on_bedrock_converse_answers_400(monkeypatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.responses( + model="bedrock/converse/us.openai.gpt-oss-120b-1:0", + input="What does the deepwiki MCP server say about litellm?", + tools=[ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", + } + ], + aws_access_key_id="AKIAHOSTEDMCP", + aws_secret_access_key="hosted-mcp-secret", + aws_region_name="us-east-1", + ) + assert raised.value.status_code == 400 + assert "`deepwiki`" in str(raised.value) + assert "litellm_proxy/mcp/" in str(raised.value) diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 2d98b29173d..ef630dd6946 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -138,6 +138,46 @@ describe("responses_api", () => { expect(onTimingData).not.toHaveBeenCalled(); }); + it.each([ + [["__all__"], [], [{ server_label: "litellm", server_url: "litellm_proxy/mcp" }]], + [ + ["server-1"], + [{ server_id: "server-1", server_name: "my server" }], + [{ server_label: "my server", server_url: "litellm_proxy/mcp/my server" }], + ], + ])( + "sends MCP selection %j as gateway references the proxy runs for any model", + async (selected, servers, expected) => { + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gemini-pro", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + selected, + undefined, // previousResponseId + undefined, // onResponseId + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + servers as never, + ); + + expect(mockResponsesCreate.mock.calls[0][0].tools).toEqual( + expected.map((tool) => ({ type: "mcp", require_approval: "never", ...tool })), + ); + }, + ); + it("should report total latency in both streaming and non-streaming modes", async () => { const onTotalLatency = vi.fn(); const callWithStreaming = (streamingEnabled: boolean) => @@ -406,14 +446,14 @@ describe("responses_api", () => { { type: "mcp", server_label: "Alpha", - server_url: "https://backend.example.com/mcp/Alpha", + server_url: "litellm_proxy/mcp/Alpha", require_approval: "never", allowed_tools: ["toolA"], }, { type: "mcp", server_label: "Beta", - server_url: "https://backend.example.com/mcp/Beta", + server_url: "litellm_proxy/mcp/Beta", require_approval: "never", allowed_tools: ["toolB", "toolC"], }, diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 1f717e488e9..82fcb699fb1 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,11 +1,12 @@ import openai from "openai"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; -import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; +import { getGatewayBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; import { parseUsageCost } from "./usage_cost"; import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers"; +import { buildMcpToolBlocks, type BuildMcpToolBlocksArgs } from "./mcp_tool_blocks"; import type { MCPEvent } from "@/components/mcp_tools/types"; import { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; import { @@ -103,7 +104,6 @@ export async function makeOpenAIResponsesRequest( } const proxyBaseUrl = customBaseUrl || getGatewayBaseUrl(); - const mcpBaseUrl = customBaseUrl || getProxyBaseUrl(); const headers = buildPlaygroundHeaders(tags, customHeaders); const client = new openai.OpenAI({ @@ -137,50 +137,13 @@ export async function makeOpenAIResponsesRequest( }); // Build tools array - const tools: any[] = []; - - // Add MCP servers if selected - if (selectedMCPServers && selectedMCPServers.length > 0) { - if (selectedMCPServers.includes("__all__")) { - // All MCP Servers selected - tools.push({ - type: "mcp", - server_label: "litellm", - server_url: `${mcpBaseUrl}/mcp`, - require_approval: "never", - }); - } else { - // Individual servers/toolsets selected - create one entry per item - selectedMCPServers.forEach((serverId) => { - if (serverId.startsWith("toolset:")) { - // Toolset: same /{name}/mcp pattern as individual servers - const toolsetId = serverId.slice("toolset:".length); - const toolset = mcpToolsets?.find((t) => t.toolset_id === toolsetId); - const toolsetName = toolset?.toolset_name || toolsetId; - tools.push({ - type: "mcp", - server_label: toolsetName, - server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(toolsetName)}`, - require_approval: "never", - }); - } else { - const server = mcpServers?.find((s) => s.server_id === serverId); - // Use server_name for both routing and labelling. server_name is the - // unique registered identifier; aliases can collide across servers. - const routeName = server?.server_name || serverId; - const allowedTools = mcpServerToolRestrictions?.[serverId] || []; - - tools.push({ - type: "mcp", - server_label: routeName, // unique per request — collisions cause silent tool-routing failures - server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(routeName)}`, - require_approval: "never", - ...(allowedTools.length > 0 ? { allowed_tools: allowedTools } : {}), - }); - } - }); - } - } + const mcpSelection: BuildMcpToolBlocksArgs = { + selectedMCPServers, + mcpServers, + mcpToolsets, + mcpServerToolRestrictions, + }; + const tools: any[] = buildMcpToolBlocks(mcpSelection); // Add code_interpreter tool if enabled (OpenAI auto-creates container) if (codeInterpreterEnabled) { From 22563a80d1cb41d954795365c596b7a1fa9e9831 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:50:30 -0800 Subject: [PATCH 11/18] fix(mcp): keep DeepSeek tool cleanup and the guardrail policy result on lts/1.108.x DeepSeek now sends the params through its own tool cleanup when a hosted MCP tool is dropped, so a tool_choice naming that tool and the tool selection params beside legacy functions are cleared as the line does today. The MCP security guardrail uses the gateway's saved toolset check, so a failed read reports the name as unregistered with the policy result instead of raising. Also fixes three lint errors the line already has, two import orders and one duplicate test import, since make check cannot pass on this line without them. --- litellm/llms/deepseek/chat/transformation.py | 7 ++++- .../redis_update_buffer.py | 1 - .../mcp_security/mcp_security_guardrail.py | 12 ++------ .../mcp/litellm_proxy_mcp_handler.py | 4 +-- litellm/utils.py | 2 +- .../test_streaming_chunk_builder_utils.py | 1 - .../chat/test_deepseek_chat_transformation.py | 30 +++++++++++++++++++ .../guardrail_hooks/test_mcp_security.py | 27 +++++++++++++++++ 8 files changed, 68 insertions(+), 16 deletions(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index e1e9694e2ab..a742cc6c4de 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -42,9 +42,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Reference: https://api-docs.deepseek.com/guides/thinking_mode """ + params_without_hosted_mcp: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route="deepseek", drop_params=drop_params + ) # Let parent handle standard params first optional_params = super().map_openai_params( - tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params), + params_without_hosted_mcp + if params_without_hosted_mcp is non_default_params + else self._drop_unsupported_tools(non_default_params), optional_params, model, drop_params, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 22f55ea9153..8fbaa7e3a9a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter - from redis.exceptions import RedisError from litellm._internal_context import with_service_target diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 9cd5555fe25..66f493bcea3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -18,6 +18,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.proxy._types import UserAPIKeyAuth from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LITELLM_PROXY_MCP_SERVER_URL_PREFIX, + saved_toolset_exists, ) from litellm.types.guardrails import GuardrailEventHooks @@ -104,7 +105,6 @@ class MCPSecurityGuardrail(CustomGuardrail): from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy.proxy_server import prisma_client async with global_mcp_server_manager.catalog.operation(): not_servers: Final = frozenset( @@ -112,12 +112,4 @@ class MCPSecurityGuardrail(CustomGuardrail): for name in requested_servers if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None ) - if prisma_client is None: - return not_servers - return frozenset( - [ - name - for name in not_servers - if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None - ] - ) + return frozenset([name for name in not_servers if not await saved_toolset_exists(name)]) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index b711d051055..1dfa4b2abcb 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -105,7 +105,7 @@ def _registry_serves(name: str, servers: Collection[MCPServer]) -> bool: ) -async def _toolset_exists(name: str) -> bool: +async def saved_toolset_exists(name: str) -> bool: try: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, @@ -124,7 +124,7 @@ async def _toolset_exists(name: str) -> bool: async def _gateway_served_names( names: Collection[str], servers: Callable[[], Collection[MCPServer]] = _registered_mcp_servers, - toolset_exists: Callable[[str], Awaitable[bool]] = _toolset_exists, + toolset_exists: Callable[[str], Awaitable[bool]] = saved_toolset_exists, ) -> frozenset[str]: registered: Final = tuple(servers()) if names else () return frozenset([name for name in names if _registry_serves(name, registered) or await toolset_exists(name)]) diff --git a/litellm/utils.py b/litellm/utils.py index fea6b9d2177..8cc0cd3ccce 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -438,11 +438,11 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, ) - from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index a46c8bed4ad..716c0af236c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -18,7 +18,6 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index ab38f4fc41f..d4775173fe0 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -952,3 +952,33 @@ def test_deepseek_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(mo assert "tools" not in result assert "tool_choice" not in result assert "parallel_tool_calls" not in result + + +def test_deepseek_clears_a_tool_choice_naming_the_hosted_mcp_tool_drop_params_dropped(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = DeepSeekChatConfig().map_openai_params( + { + "tools": [_HOSTED_MCP_TOOL, _function_tool("shell")], + "tool_choice": {"type": "function", "function": {"name": "deepwiki"}}, + }, + {}, + "deepseek-v4-flash", + True, + ) + assert result["tools"] == [_function_tool("shell")] + assert "tool_choice" not in result + + +def test_deepseek_removes_tool_selection_beside_legacy_functions_when_drop_params_drops_every_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + functions: Final = [{"name": "shell", "parameters": {"type": "object", "properties": {}}}] + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL], "functions": functions, "tool_choice": "required", "parallel_tool_calls": True}, + {}, + "deepseek-v4-flash", + True, + ) + assert result["functions"] == functions + assert "tools" not in result + assert "tool_choice" not in result + assert "parallel_tool_calls" not in result diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py index 6501526634c..eeaa105565e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py @@ -255,3 +255,30 @@ async def test_allows_saved_toolset_by_name_and_blocks_unknown_name(guardrail, c ): await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) assert exc.value.detail["unregistered_servers"] == ["evil_server"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) +async def test_blocks_the_name_with_the_policy_result_when_the_toolset_read_fails(guardrail, call_type): + prisma: Final = MagicMock() + prisma.writer_db = prisma.db + prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(side_effect=RuntimeError("toolset table unavailable")) + data: Final = { + "tools": [{"type": "mcp", "server_url": "litellm_proxy/mcp/research"}], + "guardrails": ["test-mcp-security"], + } + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + MCPServerManager(), + ), + pytest.raises(HTTPException) as exc, + ): + await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) + assert exc.value.status_code == 400 + assert exc.value.detail["unregistered_servers"] == ["research"] From 28abdf7dfe7cc02855a39f01cda064d6a5353881 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Thu, 5 Nov 2026 22:42:57 -0800 Subject: [PATCH 12/18] fix(bedrock): forward the per-turn-control beta to Bedrock Invoke (#43086) Backport of #43086 to lts/1.108.x. Cherry-picked from 08f2a98250df4ee67943d449c051a6d37e12368e (main). Conflicts resolved for the line: update_headers_with_filtered_beta keeps the line's (headers, provider) signature and gains only keep=, the messages handler passes keep= at the line's existing filter call, DEPLOYMENT_SCOPED_CAPABILITY_FIELDS adds supports_per_turn_control to the line's own set, the Invoke config (anthropic_claude3_transformation.py on the line) gains the _MODEL_INFO adapter main already had, and the test file gains main's ScriptedBedrockInvoke and _ANTHROPIC_MESSAGE helpers the new tests use. --- litellm/anthropic_beta_headers_manager.py | 17 ++- .../anthropic_messages/transformation.py | 3 + litellm/llms/bedrock/common_utils.py | 9 ++ .../anthropic_claude3_transformation.py | 30 +++- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- ...odel_prices_and_context_window_backup.json | 9 ++ litellm/types/utils.py | 5 +- litellm/utils.py | 1 + model_prices_and_context_window.json | 9 ++ .../test_anthropic_claude3_transformation.py | 142 ++++++++++++++++++ 10 files changed, 220 insertions(+), 11 deletions(-) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 96c38b69104..d57cbd6d8f5 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -354,6 +354,7 @@ def get_provider_beta_header( def update_headers_with_filtered_beta( headers: dict, provider: str, + keep: frozenset[str] = frozenset(), ) -> dict: """ Update headers dict by filtering and transforming anthropic-beta header values. @@ -362,6 +363,7 @@ def update_headers_with_filtered_beta( Args: headers: Request headers dict (will be modified in place) provider: Provider name + keep: Betas the routed model accepts, forwarded as sent whatever the row says Returns: Updated headers dict @@ -371,12 +373,19 @@ def update_headers_with_filtered_beta( return headers # Parse existing beta headers - beta_values: Final = [b.strip() for b in existing_beta.split(",") if b.strip()] + beta_values: Final[tuple[str, ...]] = tuple(b.strip() for b in existing_beta.split(",") if b.strip()) # Filter and transform based on provider - filtered_beta_values: Final = filter_and_transform_beta_headers( - beta_headers=beta_values, - provider=provider, + filtered_beta_values: Final = tuple( + dict.fromkeys( + [ + *filter_and_transform_beta_headers( + beta_headers=[b for b in beta_values if b not in keep], + provider=provider, + ), + *(b for b in beta_values if b in keep), + ] + ) ) # Update or remove the header diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bd6c53a0453..e02f7aad084 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -138,6 +138,9 @@ class BaseAnthropicMessagesConfig(ABC): """ return True + def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + return frozenset() + def handles_web_search_natively(self) -> bool: """ Whether the upstream this config routes to executes ``web_search`` tools diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 999dd46b01a..20cb32e880d 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1270,6 +1270,15 @@ def bedrock_supports_tool_search(model: str) -> bool: return AnthropicModelInfo.supports_model_capability(model, "supports_tool_search", "bedrock") +def bedrock_supports_per_turn_control(model: str, model_info: Mapping[str, object] | None) -> bool: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + deployment_flag: Final = (model_info or {}).get("supports_per_turn_control") + if isinstance(deployment_flag, bool): + return deployment_flag + return AnthropicModelInfo.supports_model_capability(model, "supports_per_turn_control", "bedrock") + + def supports_bedrock_prompt_cache_ttl(model: str) -> bool: """ Whether Bedrock accepts a ``ttl`` ("5m"/"1h") on prompt ``cache_control`` diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 00ef2673da1..28228b4c62d 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -3,6 +3,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -37,6 +38,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation from litellm.llms.bedrock.common_utils import ( BedrockError, apply_bedrock_invoke_structured_output, + bedrock_supports_per_turn_control, bedrock_supports_tool_search, ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, @@ -70,6 +72,9 @@ else: LiteLLMLoggingObj = Any +_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + class AmazonAnthropicClaudeMessagesConfig( AnthropicMessagesConfig, AmazonInvokeConfig, @@ -91,6 +96,13 @@ class AmazonAnthropicClaudeMessagesConfig( def beta_headers_provider(self) -> str: return self.custom_llm_provider or "bedrock" + def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]: + if bedrock_supports_per_turn_control( + model, _MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)) + ): + return frozenset({ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value}) + return frozenset() + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def get_error_class( @@ -517,6 +529,7 @@ class AmazonAnthropicClaudeMessagesConfig( headers: dict, anthropic_messages_request: dict, injected_thinking_for_clear_thinking: bool, + litellm_params: GenericLiteLLMParams, ) -> list[str]: anthropic_model_info: Final = AnthropicModelInfo() tools: Final = anthropic_messages_optional_request_params.get("tools") @@ -578,16 +591,22 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") + model_supported_betas: Final = self.model_specific_anthropic_betas(model, litellm_params) & user_beta_set beta_provider: Final = self.beta_headers_provider filtered_betas: Final = sorted( - filter_and_transform_beta_headers( - beta_headers=list(beta_set), - provider=beta_provider, - ) + { + *filter_and_transform_beta_headers( + beta_headers=list(beta_set), + provider=beta_provider, + ), + *model_supported_betas, + } ) dropped_user_betas: Final = sorted( - b for b in user_beta_set if not filter_and_transform_beta_headers([b], provider=beta_provider) + b + for b in user_beta_set - model_supported_betas + if not filter_and_transform_beta_headers([b], provider=beta_provider) ) if dropped_user_betas: verbose_logger.warning( @@ -770,6 +789,7 @@ class AmazonAnthropicClaudeMessagesConfig( headers=headers, anthropic_messages_request=anthropic_messages_request, injected_thinking_for_clear_thinking=injected_thinking_for_clear_thinking, + litellm_params=litellm_params, ) if filtered_betas: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3eff8bc3b18..4a21a93334c 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2270,7 +2270,11 @@ class BaseLLMHTTPHandler: ) if anthropic_messages_provider_config.should_filter_anthropic_beta_headers(): - headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider) + headers = update_headers_with_filtered_beta( + headers=headers, + provider=custom_llm_provider, + keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params), + ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2c183527ce8..891e247747e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -64002,6 +64002,15 @@ "supports_tool_search": true } }, + { + "name": "claude-per-turn-control", + "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "fill_missing_for_providers": ["bedrock", "bedrock_converse"], + "description": "Claude at version 4.5 or higher, with the same id shapes and version boundaries as claude-tool-search. On 2026-11-05 Bedrock InvokeModel answered 200 with the per-turn-control-2026-07-01 beta on every Claude 4.5 and newer model a QA account reached across 13 regions, and 400 invalid beta flag on Claude 3 Haiku in ca-central-1. The jp., au. and in. profiles answered 200 too, AWS lists no apac. profile for Claude 4.5 or newer, and GovCloud was not tested. Claude 3.5, 3.7, Sonnet 4, Opus 4 and Opus 4.1 were unreachable, so they keep the beta dropped as before. The flag means Bedrock accepts the beta name on the model, as supports_tool_search does for the tool search beta, and Bedrock Invoke forwards the beta only where it is set. It does not promise per-turn output_config works, since on 2026-11-05 Opus 4.8, Opus 5, Sonnet 5 and Fable 5 rejected it with or without the beta. A deployment turns the forward off with model_info supports_per_turn_control false.", + "model_info": { + "supports_per_turn_control": true + } + }, { "name": "wandb-reasoning-baseline", "pattern": "^wandb/", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 77160e0adf1..6a5e195c412 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -212,6 +212,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] supports_tool_search: bool | None + supports_per_turn_control: ReadOnly[bool | None] supports_mid_conversation_system: bool | None supports_url_context: bool | None supports_none_reasoning_effort: bool | None @@ -4083,7 +4084,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) -DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"}) +DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( + {"supports_regex_lookaround", "supports_per_turn_control"} +) DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..e0c365f1a81 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6749,6 +6749,7 @@ def get_model_info_helper( supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), supports_tool_search=_model_info.get("supports_tool_search", None), + supports_per_turn_control=_model_info.get("supports_per_turn_control", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2c183527ce8..891e247747e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -64002,6 +64002,15 @@ "supports_tool_search": true } }, + { + "name": "claude-per-turn-control", + "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "fill_missing_for_providers": ["bedrock", "bedrock_converse"], + "description": "Claude at version 4.5 or higher, with the same id shapes and version boundaries as claude-tool-search. On 2026-11-05 Bedrock InvokeModel answered 200 with the per-turn-control-2026-07-01 beta on every Claude 4.5 and newer model a QA account reached across 13 regions, and 400 invalid beta flag on Claude 3 Haiku in ca-central-1. The jp., au. and in. profiles answered 200 too, AWS lists no apac. profile for Claude 4.5 or newer, and GovCloud was not tested. Claude 3.5, 3.7, Sonnet 4, Opus 4 and Opus 4.1 were unreachable, so they keep the beta dropped as before. The flag means Bedrock accepts the beta name on the model, as supports_tool_search does for the tool search beta, and Bedrock Invoke forwards the beta only where it is set. It does not promise per-turn output_config works, since on 2026-11-05 Opus 4.8, Opus 5, Sonnet 5 and Fable 5 rejected it with or without the beta. A deployment turns the forward off with model_info supports_per_turn_control false.", + "model_info": { + "supports_per_turn_control": true + } + }, { "name": "wandb-reasoning-baseline", "pattern": "^wandb/", diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 30bbc7e5179..d36392927f8 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -6,6 +6,7 @@ import os import struct import zlib from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from types import SimpleNamespace from typing import Final @@ -2309,6 +2310,32 @@ def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cos ] +def test_bedrock_invoke_transform_forwards_per_turn_control_beta_with_message_output_config( + local_model_cost_map: None, local_beta_headers_config: None +) -> None: + """A live InvokeModel call on 2026-11-05 answered 200 with per-turn-control-2026-07-01 next to a message + ``output_config`` on Opus 5.5, so the client's beta and that message must both reach Bedrock.""" + from litellm.types.router import GenericLiteLLMParams + + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": [], "output_config": {"effort": "low"}}, + {"role": "user", "content": "continue"}, + ] + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="us.anthropic.claude-opus-5-5", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": "per-turn-control-2026-07-01"}, + ) + + assert result["messages"] == messages + assert result["anthropic_beta"].count("per-turn-control-2026-07-01") == 1 + + def test_bedrock_invoke_transform_converts_mid_conversation_system_for_older_claude(local_model_cost_map): """Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, Haiku 4.5, etc. ("role 'system' is not supported on this model"), but @@ -3837,6 +3864,121 @@ def test_bedrock_removed_tool_change_does_not_add_beta(explicit_beta: bool) -> N assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta) +_ANTHROPIC_MESSAGE: Final = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": [{"type": "text", "text": '{"summary": "sunny", "kind": 1}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} + + +@dataclass(slots=True) +class ScriptedBedrockInvoke: + replies: tuple[tuple[int, Mapping[str, object] | bytes], ...] + bodies: tuple[dict[str, object], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.bodies = (*self.bodies, REASONING_BODY_ADAPTER.validate_json(request.content)) + status_code, body = self.replies[len(self.bodies) - 1] + if isinstance(body, bytes): + return httpx.Response(status_code, content=body) + return httpx.Response(status_code, json=body) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize( + ("model", "headers", "forwarded"), + [ + ("anthropic.claude-3-haiku-20240307-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, False), + ("us.anthropic.claude-opus-4-1-20250805-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, False), + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("jp.anthropic.claude-opus-4-6-v1", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("au.anthropic.claude-opus-4-8", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("eu.anthropic.claude-opus-5-5", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("global.anthropic.claude-sonnet-5-5", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("us.anthropic.claude-newfamily-6", {"anthropic-beta": "per-turn-control-2026-07-01"}, True), + ("us.anthropic.claude-sonnet-5-5", {}, False), + ], +) +async def test_bedrock_invoke_messages_forward_per_turn_control_beta_only_where_the_model_map_flags_it( + model: str, headers: dict[str, str], forwarded: bool +) -> None: + """On 2026-11-05 Bedrock InvokeModel answered 400 ``invalid beta flag`` to per-turn-control-2026-07-01 on Claude 3 + Haiku and 200 on every Claude 4.5 and newer model a QA account reached in 13 regions and in the jp., au. and in. + profiles.""" + bedrock: Final = ScriptedBedrockInvoke(replies=((200, _ANTHROPIC_MESSAGE),)) + + await litellm.anthropic_messages( + model=f"bedrock/invoke/{model}", + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + extra_headers=headers, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), + ) + + (sent,) = bedrock.bodies + sent_betas: Final = TypeAdapter(tuple[str, ...]).validate_python(sent.get("anthropic_beta", ())) + assert ("per-turn-control-2026-07-01" in sent_betas) is forwarded + + +@pytest.mark.parametrize( + ("model", "flag", "expected"), + [ + ("us.anthropic.claude-opus-5-5", False, (False, True)), + ("anthropic.claude-3-haiku-20240307-v1:0", True, (True, False)), + ], + ids=["opt_out", "opt_in"], +) +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +async def test_bedrock_invoke_messages_per_turn_control_flag_stays_on_its_own_deployment( + respx_mock: respx.Router, + monkeypatch: pytest.MonkeyPatch, + model: str, + flag: bool, + expected: tuple[bool, bool], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + bedrock: Final = ScriptedBedrockInvoke(replies=((200, _ANTHROPIC_MESSAGE), (200, _ANTHROPIC_MESSAGE))) + respx_mock.post(url__regex=r"https://bedrock-runtime\..*").mock(side_effect=bedrock) + router: Final = litellm.Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": { + "model": f"bedrock/invoke/{model}", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": model_info, + } + for model_name, model_info in (("flagged", {"supports_per_turn_control": flag}), ("default", {})) + ], + num_retries=0, + ) + + for model_name in ("flagged", "default"): + await router.aanthropic_messages( + model=model_name, + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + extra_headers={"anthropic-beta": "per-turn-control-2026-07-01"}, + ) + + forwarded: Final = tuple( + "per-turn-control-2026-07-01" in TypeAdapter(tuple[str, ...]).validate_python(sent.get("anthropic_beta", ())) + for sent in bedrock.bodies + ) + assert forwarded == expected + + @pytest.mark.parametrize("trace", ["enabled", "disabled"]) def test_bedrock_messages_applies_the_deployment_guardrail_config_as_invoke_headers(trace: str) -> None: from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( From 8a60580280eb41fae3e6461b05a3c5c0fd44a4ce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:57:46 -0800 Subject: [PATCH 13/18] fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved Converse used to drop any service_tier outside default, flex, and priority with no error, rewrite auto to default, and drop reserved. Now reserved and standard_only map to Bedrock tiers, auto sends no tier, and any other value answers 400 unless drop_params is on. Nova batch rows honor the row's drop_params. Backport to lts/1.108.x of ce94d2894d5 (litellm_bedrock_converse_service_tier_values), whose main PR is pending. Adapted to the line: lts has no REQUEST_PARAMS, so the call passes the value through TypeAdapter(object) and keeps the line's two-argument requestMetadata call. --- .../bedrock/chat/converse_transformation.py | 33 +++-- litellm/llms/bedrock/files/transformation.py | 2 +- litellm/types/llms/bedrock.py | 2 +- .../llms/bedrock/chat/test_service_tier.py | 126 +++++++++++++++++- .../test_bedrock_files_transformation.py | 35 ++++- 5 files changed, 176 insertions(+), 22 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6984fc1c1cd..4f4ba66088e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -160,6 +160,10 @@ def _tools_the_model_accepts( return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools] +BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved") +_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard_only": "default"}) + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -1200,8 +1204,8 @@ class AmazonConverseConfig(BaseConfig): self._map_context_management_param(value, optional_params) if param == "requestMetadata": self._map_request_metadata_param(value, optional_params) - if param == "service_tier" and isinstance(value, str): - self._map_service_tier_param(value, optional_params) + if param == "service_tier": + self._map_service_tier_param(_JSON_VALUE.validate_python(value), optional_params, model, drop_params) if param == "web_search_options" and isinstance(value, dict): # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` @@ -1254,17 +1258,22 @@ class AmazonConverseConfig(BaseConfig): if mapped is not None: optional_params["context_management"] = mapped - def _map_service_tier_param(self, value: str, optional_params: dict) -> None: - """Map OpenAI service_tier (string) to Bedrock serviceTier (object). - - OpenAI values: "auto", "default", "flex", "priority" - Bedrock values: "default", "flex", "priority" (no "auto") - """ - bedrock_tier = value - if value == "auto": - bedrock_tier = "default" # Bedrock doesn't support "auto" - if bedrock_tier in ("default", "flex", "priority"): + def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None: + if value is None or value == "auto": + return + bedrock_tier: Final = _BEDROCK_SERVICE_TIER_ALIASES.get(value, value) if isinstance(value, str) else None + if bedrock_tier in BEDROCK_SERVICE_TIERS: optional_params["serviceTier"] = {"type": bedrock_tier} + return + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={value!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.warning("Dropping service_tier=%r for %s, since Bedrock does not serve it.", value, model) def _translate_response_format_param( self, diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d40e0dec85..7d3ac50faa0 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -1024,7 +1024,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): non_default_params=optional_params, optional_params={}, model=model, - drop_params=False, + drop_params=optional_params.get("drop_params") is True, ) return converse_config.transform_request( model=model, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index ff4f2c4fd1e..e7bef9b3946 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False): class ServiceTierBlock(TypedDict): - type: Literal["priority", "default", "flex"] + type: Literal["priority", "default", "flex", "reserved"] class ConverseResponseBlock(TypedDict, total=False): diff --git a/tests/unit/llms/bedrock/chat/test_service_tier.py b/tests/unit/llms/bedrock/chat/test_service_tier.py index ce9dc4d745e..e248a64fba4 100644 --- a/tests/unit/llms/bedrock/chat/test_service_tier.py +++ b/tests/unit/llms/bedrock/chat/test_service_tier.py @@ -3,11 +3,13 @@ Tests for Bedrock Converse API serviceTier support. """ import json +from unittest.mock import MagicMock import pytest - +import litellm from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.llms.bedrock import ServiceTierBlock @@ -201,11 +203,8 @@ def test_map_openai_service_tier_flex(): assert result["serviceTier"] == {"type": "flex"} -def test_map_openai_service_tier_auto_maps_to_default(): - """Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'. - - Bedrock doesn't support 'auto', so we map it to 'default'. - """ +def test_map_openai_service_tier_auto_sends_no_tier(): + """`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default.""" config = AmazonConverseConfig() result = config.map_openai_params( @@ -215,10 +214,123 @@ def test_map_openai_service_tier_auto_maps_to_default(): drop_params=False, ) - assert "serviceTier" in result + assert "serviceTier" not in result + + +def test_map_openai_service_tier_none_sends_no_tier(): + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": None}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_reserved_reaches_the_request(): + config = AmazonConverseConfig() + model = "anthropic.claude-3-sonnet-20240229-v1:0" + + optional_params = config.map_openai_params( + non_default_params={"service_tier": "reserved"}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["serviceTier"] == {"type": "reserved"} + + +def test_map_openai_service_tier_standard_only_maps_to_default(): + """`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier.""" + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "standard_only"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + assert result["serviceTier"] == {"type": "default"} +@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1]) +def test_map_openai_service_tier_unmapped_value_answers_400(value): + config = AmazonConverseConfig() + optional_params = {} + + with pytest.raises(litellm.utils.UnsupportedParamsError) as error: + config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params=optional_params, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert error.value.status_code == 400 + assert f"service_tier={value!r}" in error.value.message + assert "drop_params" in error.value.message + assert "serviceTier" not in optional_params + + +@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}]) +def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value): + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=True, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", True) + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "ultrafast"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"]) +def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + client = MagicMock(spec=HTTPHandler) + + with pytest.raises(litellm.BadRequestError) as error: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + service_tier="ultrafast", + aws_region_name="us-east-1", + client=client, + ) + + assert error.value.status_code == 400 + assert "service_tier='ultrafast'" in error.value.message + client.post.assert_not_called() + + # Tests for service_tier in response diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 425beefac8a..5dc242386d3 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -22,7 +22,7 @@ from botocore.credentials import Credentials import litellm from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET -from litellm.llms.bedrock.files.transformation import BedrockJsonlFilesTransformation +from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation from tests.unit.proxy.conftest import httpx_transport @@ -101,6 +101,39 @@ class TestBedrockFilesTransformation: assert "_litellm_undeclared_sentinel" not in model_input, model_input assert result[0]["modelInput"]["max_tokens"] == 10 + @staticmethod + def _nova_row(**body_params: object) -> dict[str, object]: + return { + "custom_id": "nova-tier-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [{"role": "user", "content": "hi"}], + **body_params, + }, + } + + def test_nova_row_with_null_service_tier_sends_no_tier(self): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier=None)] + ) + + assert "serviceTier" not in result[0]["modelInput"] + + def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self): + with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"): + BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier="ultrafast")] + ) + + def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier="ultrafast", drop_params=True)] + ) + + assert "serviceTier" not in result[0]["modelInput"] + def test_nova_text_only_uses_converse_format(self): """ Test that Nova models produce Converse API format in batch modelInput. From f2cd45a8abe5dcfd318912b28ab83c80b8f981f9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:02:55 -0800 Subject: [PATCH 14/18] style: clear the ruff reds lts/1.108.x already has Sorts two imports ruff flags (I001) in redis_update_buffer.py and utils.py, the same hunks #50348 carries, and drops a duplicate GenericStreamingChunk import (F811) in a streaming test. No behavior change. --- litellm/proxy/db/db_transaction_queue/redis_update_buffer.py | 1 - litellm/utils.py | 2 +- .../litellm_core_utils/test_streaming_chunk_builder_utils.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 22f55ea9153..8fbaa7e3a9a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter - from redis.exceptions import RedisError from litellm._internal_context import with_service_target diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..71c9ca0a16e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -434,11 +434,11 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, ) - from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index a46c8bed4ad..716c0af236c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -18,7 +18,6 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, From a97503a3930e9f36261fc76ab3622f05b9856f2d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:37:23 -0800 Subject: [PATCH 15/18] fix(bedrock): keep a responses batch row's drop_params A /v1/responses batch row went through the Responses-to-Chat bridge, which keeps only Responses fields, so the row's own drop_params was lost while its service_tier still reached the Nova mapper. An unknown tier then failed the file upload even with drop_params: true on the row. The row's drop_params now rides along to the mapper, and the passthrough branch keeps it out of the modelInput sent to Bedrock. --- litellm/llms/bedrock/files/transformation.py | 8 +++-- .../test_bedrock_files_transformation.py | 32 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 7d3ac50faa0..f0640410b4b 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -942,7 +942,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model), so `input`, `instructions`, `max_output_tokens` and the tool params translate identically in batch and real time. """ - return responses_batch_body_to_chat_body(openai_request_body) + chat_body: Final = responses_batch_body_to_chat_body(openai_request_body) + drop_params: Final = openai_request_body.get("drop_params") + if drop_params is None: + return chat_body + return _frozen_mapping(chain(chat_body.items(), (("drop_params", drop_params),))) @staticmethod def _transform_batch_body_to_chat_body( @@ -1038,7 +1042,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # like openai.gpt-oss-*, qwen, deepseek, etc.) --- return { "messages": messages, - **optional_params, + **{k: v for k, v in optional_params.items() if k != "drop_params"}, } def _resolve_batch_record_model_and_provider( diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 5dc242386d3..d82d01fb6b2 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -102,16 +102,15 @@ class TestBedrockFilesTransformation: assert result[0]["modelInput"]["max_tokens"] == 10 @staticmethod - def _nova_row(**body_params: object) -> dict[str, object]: + def _nova_row( + url: str = "/v1/chat/completions", model: str = "us.amazon.nova-pro-v1:0", **body_params: object + ) -> dict[str, object]: + prompt: Final = {"input": "hi"} if url == "/v1/responses" else {"messages": [{"role": "user", "content": "hi"}]} return { "custom_id": "nova-tier-1", "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "us.amazon.nova-pro-v1:0", - "messages": [{"role": "user", "content": "hi"}], - **body_params, - }, + "url": url, + "body": {"model": model, **prompt, **body_params}, } def test_nova_row_with_null_service_tier_sends_no_tier(self): @@ -121,18 +120,29 @@ class TestBedrockFilesTransformation: assert "serviceTier" not in result[0]["modelInput"] - def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self): + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self, url: str): with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"): BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( - [self._nova_row(service_tier="ultrafast")] + [self._nova_row(url, service_tier="ultrafast")] ) - def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self): + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self, url: str): result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( - [self._nova_row(service_tier="ultrafast", drop_params=True)] + [self._nova_row(url, service_tier="ultrafast", drop_params=True)] ) assert "serviceTier" not in result[0]["modelInput"] + assert "drop_params" not in json.dumps(result[0]["modelInput"]) + + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_passthrough_row_keeps_row_level_drop_params_out_of_the_model_input(self, url: str): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, "openai.gpt-oss-120b-1:0", drop_params=True)] + ) + + assert "drop_params" not in result[0]["modelInput"] def test_nova_text_only_uses_converse_format(self): """ From 7771cbb1ec8ad8b0bf29ff305f31b13d4ab18cd5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:02:55 -0800 Subject: [PATCH 16/18] style: clear the ruff reds lts/1.108.x already has Sorts two imports ruff flags (I001) in redis_update_buffer.py and utils.py, the same hunks #50348 carries, and drops a duplicate GenericStreamingChunk import (F811) in a streaming test. No behavior change. --- litellm/proxy/db/db_transaction_queue/redis_update_buffer.py | 1 - litellm/utils.py | 2 +- .../litellm_core_utils/test_streaming_chunk_builder_utils.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 22f55ea9153..8fbaa7e3a9a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter - from redis.exceptions import RedisError from litellm._internal_context import with_service_target diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..71c9ca0a16e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -434,11 +434,11 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, ) - from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index a46c8bed4ad..716c0af236c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -18,7 +18,6 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, From eb25b1794610c90a9bc62adba4c11815fe83e805 Mon Sep 17 00:00:00 2001 From: Julien Parenti Date: Fri, 6 Nov 2026 02:41:12 -0800 Subject: [PATCH 17/18] fix(vertex_ai): honor vertex_location for the Cloud Text-to-Speech endpoint (#50255) Backport of #50255 to lts/1.108.x. Cherry-picked from 6fc9555f83b8130a0ffd1e18f38b165eebbebe75 (main), adapted by hand: the line's text-to-speech module has no Gemini TTS code, so the Gemini parts of the pick are dropped and chirp takes its host from the deployment's or call's own vertex_location only. --- .../text_to_speech/transformation.py | 45 ++++- .../text_to_speech/test_transformation.py | 173 +++++++++++++++++- 2 files changed, 214 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2fae2a9de1c..eaaa600b870 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -49,6 +49,43 @@ _LyriaVoice: TypeAlias = str | dict | None _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_ARRAY: Final = TypeAdapter(tuple[object, ...], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str, config=ConfigDict(hide_input_in_errors=True)) +_CLOUD_TTS_LOCATIONS: Final = MappingProxyType( + { + **dict.fromkeys( + ( + "us", + "us-central1", + "us-east1", + "us-east4", + "us-east5", + "us-south1", + "us-west1", + "us-west2", + "us-west3", + "us-west4", + "us-west5", + ), + "us", + ), + **dict.fromkeys( + ( + "eu", + "europe-central2", + "europe-north1", + "europe-north2", + "europe-southwest1", + "europe-west1", + "europe-west3", + "europe-west4", + "europe-west8", + "europe-west9", + "europe-west10", + "europe-west12", + ), + "eu", + ), + } +) class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): @@ -173,7 +210,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): # Resolve Vertex AI credentials using VertexBase helpers vertex_credentials: Final = self.safe_get_vertex_ai_credentials(litellm_params_dict) vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params_dict) - vertex_location: Final = self.safe_get_vertex_ai_location(litellm_params_dict) + vertex_location: Final = self.explicit_vertex_ai_location(litellm_params_dict) # Convert voice to string if it's a dict (extract name) # Actual voice mapping happens in map_openai_params @@ -314,7 +351,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): if api_base: return api_base - return self.TTS_API_URL + location: Final = _JSON_OBJECT.validate_python(litellm_params).get("vertex_location") + cloud_tts_location: Final = _CLOUD_TTS_LOCATIONS.get(location) if isinstance(location, str) else None + if cloud_tts_location is None: + return self.TTS_API_URL + return f"https://{cloud_tts_location}-texttospeech.googleapis.com/v1/text:synthesize" def _validate_vertex_input( self, diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index dc6f35fcfe5..b4ccb1cf081 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -48,6 +48,73 @@ class TestVertexAITextToSpeechConfig: assert url == custom_url + @pytest.mark.parametrize( + ("vertex_location", "expected_host"), + [ + ("eu", "eu-texttospeech.googleapis.com"), + ("europe-west4", "eu-texttospeech.googleapis.com"), + ("europe-west1", "eu-texttospeech.googleapis.com"), + ("europe-west3", "eu-texttospeech.googleapis.com"), + ("europe-west8", "eu-texttospeech.googleapis.com"), + ("europe-west9", "eu-texttospeech.googleapis.com"), + ("europe-west12", "eu-texttospeech.googleapis.com"), + ("europe-west10", "eu-texttospeech.googleapis.com"), + ("europe-central2", "eu-texttospeech.googleapis.com"), + ("europe-north1", "eu-texttospeech.googleapis.com"), + ("europe-north2", "eu-texttospeech.googleapis.com"), + ("europe-southwest1", "eu-texttospeech.googleapis.com"), + ("us", "us-texttospeech.googleapis.com"), + ("us-central1", "us-texttospeech.googleapis.com"), + ("us-east1", "us-texttospeech.googleapis.com"), + ("us-east4", "us-texttospeech.googleapis.com"), + ("us-east5", "us-texttospeech.googleapis.com"), + ("us-south1", "us-texttospeech.googleapis.com"), + ("us-west1", "us-texttospeech.googleapis.com"), + ("us-west2", "us-texttospeech.googleapis.com"), + ("us-west3", "us-texttospeech.googleapis.com"), + ("us-west4", "us-texttospeech.googleapis.com"), + ("us-west5", "us-texttospeech.googleapis.com"), + ], + ) + def test_get_complete_url_uses_regional_endpoint_for_vertex_location( + self, vertex_location: str, expected_host: str + ) -> None: + config: Final = VertexAITextToSpeechConfig() + + url: Final = config.get_complete_url( + model="vertex_ai/chirp", + api_base=None, + litellm_params={"vertex_location": vertex_location}, + ) + + assert url == f"https://{expected_host}/v1/text:synthesize" + + @pytest.mark.parametrize( + "vertex_location", ["global", "europe-west2", "asia-northeast1", "northamerica-northeast1", None] + ) + def test_get_complete_url_keeps_global_endpoint_without_a_regional_match(self, vertex_location: str | None) -> None: + config: Final = VertexAITextToSpeechConfig() + + url: Final = config.get_complete_url( + model="vertex_ai/chirp", + api_base=None, + litellm_params={"vertex_location": vertex_location}, + ) + + assert url == "https://texttospeech.googleapis.com/v1/text:synthesize" + + def test_get_complete_url_api_base_wins_over_vertex_location(self) -> None: + config: Final = VertexAITextToSpeechConfig() + + custom_url: Final = "https://custom-tts-endpoint.example.com/v1/synthesize" + url: Final = config.get_complete_url( + model="vertex_ai/chirp", + api_base=custom_url, + litellm_params={"vertex_location": "eu"}, + ) + + assert url == custom_url + @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): @@ -616,8 +683,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ mock_post.assert_called_once() call_kwargs = mock_post.call_args.kwargs - # Verify the URL is the Google Cloud TTS API - assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" + assert call_kwargs["url"] == "https://us-texttospeech.googleapis.com/v1/text:synthesize" # Verify request body structure assert "json" in call_kwargs @@ -764,6 +830,109 @@ async def test_aspeech_vertex_ai_sends_ssml_input( } +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_chirp_keeps_the_global_host_when_only_vertexai_location_is_set( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("VERTEXAI_LOCATION", "europe-west4") + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(b"global-audio").decode()}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/chirp", + input="Bonjour", + voice="fr-FR-Chirp3-HD-Aoede", + vertex_credentials=_AUTHORIZED_USER, + ) + + assert route.call_count == 1 + assert response.content == b"global-audio" + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_chirp_keeps_the_global_host_with_other_location_defaults( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("VERTEX_LOCATION", "europe-west4") + monkeypatch.setattr(litellm, "vertex_location", "us-central1") + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(b"global-audio").decode()}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/chirp", + input="Bonjour", + voice="fr-FR-Chirp3-HD-Aoede", + vertex_credentials=_AUTHORIZED_USER, + ) + + assert route.call_count == 1 + assert response.content == b"global-audio" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("location_param", "location", "expected_host"), + [ + ("vertex_location", "eu", "eu-texttospeech.googleapis.com"), + ("vertex_location", "us-central1", "us-texttospeech.googleapis.com"), + ("vertex_ai_location", "europe-west4", "eu-texttospeech.googleapis.com"), + ], +) +async def test_aspeech_vertex_ai_chirp_calls_the_regional_host_for_an_explicit_location( + respx_mock: respx.MockRouter, + google_token_endpoint: responses.RequestsMock, + location_param: str, + location: str, + expected_host: str, +) -> None: + route: Final = respx_mock.post(f"https://{expected_host}/v1/text:synthesize").mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(b"regional-audio").decode()}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/chirp", + input="Bonjour", + voice="fr-FR-Chirp3-HD-Aoede", + vertex_credentials=_AUTHORIZED_USER, + **{location_param: location}, + ) + + assert route.call_count == 1 + assert response.content == b"regional-audio" + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_lyria_predict_still_uses_vertexai_location( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + route: Final = respx_mock.post( + "https://us-east5-aiplatform.googleapis.com/v1/projects/test-project/locations/us-east5" + "/publishers/google/models/lyria-002:predict" + ).mock( + return_value=httpx.Response( + 200, + json={ + "predictions": [ + {"audioContent": base64.b64encode(b"lyria-audio").decode(), "mimeType": "audio/wav"}, + ] + }, + ) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/lyria-002", + input="calm piano", + voice="unused", + vertex_credentials=_AUTHORIZED_USER, + ) + + assert route.call_count == 1 + assert response.content == b"lyria-audio" + + _WAV_AUDIO_B64: Final = "UklGRiQAAABXQVZFZm10IA==" _WAV_AUDIO: Final = b"RIFF$\x00\x00\x00WAVEfmt " _UNLABELED_AUDIO_B64: Final = "bHlyaWEtMy1hdWRpbw==" From d6cfa6f0d39611a4650f640f399a538cf5dc5850 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:40:50 -0800 Subject: [PATCH 18/18] chore(deps): bump oauthlib 4.0.0 -> 4.0.1 on lts/1.108.x Lock-only bump that clears GHSA-3jp2-3mrg-2vxc and the line's osv-scan red. main already resolves oauthlib 4.0.1 (#49204), and the lines match main's and #50326's. --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 674ffa049ef..ba0f16119ff 100644 --- a/uv.lock +++ b/uv.lock @@ -6194,11 +6194,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "4.0.0" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" }, ] [[package]]