diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 98b792efa59..01ca1ac9893 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -6,6 +6,10 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union import httpx from litellm._logging import verbose_logger +from litellm.types.internal_params import ( + LITELLM_INTERNAL_REQUEST_BODY_PARAMS, + MCP_INTERNAL_PARAMS, +) from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -436,10 +440,11 @@ def filter_internal_params( data: dict, additional_internal_params: Optional[set] = None ) -> dict: """ - Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs. + Filter out LiteLLM internal MCP-handler parameters that shouldn't be re-dispatched. - This removes internal/MCP-related parameters that are used by LiteLLM internally - but should not be included in API requests to providers. + Used on completion kwargs (e.g. fallbacks) where the goal is to drop runtime + handler state before re-invoking, not to sanitize a serialized request body. + For the request-body boundary use `strip_internal_params_from_request_body`. Args: data: Dictionary of parameters to filter @@ -451,21 +456,31 @@ def filter_internal_params( if not isinstance(data, dict): return data - # Known internal parameters that should never be sent to provider APIs - internal_params = { - "skip_mcp_handler", - "mcp_handler_context", - "_skip_mcp_handler", - } + internal_params = ( + MCP_INTERNAL_PARAMS | additional_internal_params + if additional_internal_params + else MCP_INTERNAL_PARAMS + ) - # Add any additional internal params if provided - if additional_internal_params: - internal_params.update(additional_internal_params) - - # Filter out internal parameters return {k: v for k, v in data.items() if k not in internal_params} +def strip_internal_params_from_request_body(data: dict) -> dict: + """ + Remove every LiteLLM-internal optional_params key from a provider request body. + + Applied at the serialization boundary (where optional_params becomes a request + body) so internal control knobs can never reach a provider that rejects unknown + fields. See `litellm.types.internal_params.LiteLLMInternalParam` for the registry. + """ + if not isinstance(data, dict): + return data + + return { + k: v for k, v in data.items() if k not in LITELLM_INTERNAL_REQUEST_BODY_PARAMS + } + + def redact_nested_match_and_regex_keys( payload: Union[dict, List[Any], str, None], ) -> Union[dict, List[Any], str, None]: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 8fc2375c224..622df22e3f2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -8,7 +8,10 @@ import httpx import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import ( + map_finish_reason, + strip_internal_params_from_request_body, +) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.prompt_templates.factory import ( cohere_message_pt, @@ -162,7 +165,9 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): provider=provider, custom_prompt_dict=custom_prompt_dict, ) - inference_params = copy.deepcopy(optional_params) + inference_params = strip_internal_params_from_request_body( + copy.deepcopy(optional_params) + ) inference_params = { k: v for k, v in inference_params.items() diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index b6aa99842d7..06e6e1db05f 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -11,6 +11,9 @@ import httpx import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -427,7 +430,9 @@ class BedrockEmbedding(BaseAWSLLM): f"Unable to determine bedrock embedding provider for model: {model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) - inference_params = copy.deepcopy(optional_params) + inference_params = strip_internal_params_from_request_body( + copy.deepcopy(optional_params) + ) inference_params = { k: v for k, v in inference_params.items() diff --git a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py index 87ef469beb5..1b227bfcb7f 100644 --- a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py @@ -14,6 +14,9 @@ from litellm.types.llms.bedrock import ( AmazonNovaCanvasTextToImageRequest, AmazonNovaCanvasTextToImageResponse, ) +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.utils import ImageResponse @@ -73,7 +76,10 @@ class AmazonNovaCanvasConfig: # Following the same pattern as chat completions and embeddings unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841 - image_generation_config = {**image_generation_config, **optional_params} + image_generation_config = { + **image_generation_config, + **strip_internal_params_from_request_body(optional_params), + } if task_type == "TEXT_IMAGE": text_to_image_params: Dict[str, Any] = image_generation_config.pop( "textToImageParams", {} diff --git a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py index 1d88aaf35f7..14131317caf 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py @@ -5,6 +5,9 @@ from typing import List, Optional from openai.types.image import Image +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.utils import ImageResponse @@ -99,7 +102,9 @@ class AmazonStabilityConfig: text: str, optional_params: dict, ) -> dict: - inference_params = copy.deepcopy(optional_params) + inference_params = strip_internal_params_from_request_body( + copy.deepcopy(optional_params) + ) inference_params.pop( "user", None ) # make sure user is not passed in for bedrock call diff --git a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py index 8aff24fe9a7..25441dbcfd4 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py @@ -8,6 +8,9 @@ from litellm.types.llms.bedrock import ( AmazonStability3TextToImageRequest, AmazonStability3TextToImageResponse, ) +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.utils import ImageResponse @@ -73,7 +76,9 @@ class AmazonStability3Config: """ Transform the request body for the Stability 3 models """ - data = AmazonStability3TextToImageRequest(prompt=text, **optional_params) + data = AmazonStability3TextToImageRequest( + prompt=text, **strip_internal_params_from_request_body(optional_params) + ) return data @classmethod diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 93b6c563dc1..f0553fa9ed9 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -8,6 +8,9 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.image_variations.transformation import ( BaseImageVariationConfig, @@ -367,7 +370,7 @@ class BaseLLMAIOHTTPHandler: data = provider_config.transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=strip_internal_params_from_request_body(optional_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8ac5b47c6e7..4aa3202c09f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -25,6 +25,9 @@ import litellm.types.utils from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.litellm_core_utils.core_helpers import ( + strip_internal_params_from_request_body, +) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -444,7 +447,7 @@ class BaseLLMHTTPHandler: data = provider_config.transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=strip_internal_params_from_request_body(optional_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/types/internal_params.py b/litellm/types/internal_params.py new file mode 100644 index 00000000000..ae4a9fdfa08 --- /dev/null +++ b/litellm/types/internal_params.py @@ -0,0 +1,32 @@ +from enum import Enum + + +class LiteLLMInternalParam(str, Enum): + """optional_params keys LiteLLM consumes internally and must never serialize into a provider request body. + + Strict-schema providers (Bedrock and a growing set of others) reject unknown + fields with a hard 400, so any of these leaking into the wire payload fails + the whole request. This enum is the single source of truth: the request-body + filter derives its key set here, so a newly added internal knob is covered by + adding one member instead of remembering to pop it at every splat site. + """ + + SKIP_MCP_HANDLER = "skip_mcp_handler" + PRIVATE_SKIP_MCP_HANDLER = "_skip_mcp_handler" + MCP_HANDLER_CONTEXT = "mcp_handler_context" + STREAM_CHUNK_SIZE = "stream_chunk_size" + FAKE_STREAM = "fake_stream" + CACHE_CONTROL_INJECTION_POINTS = "cache_control_injection_points" + + +LITELLM_INTERNAL_REQUEST_BODY_PARAMS: frozenset[str] = frozenset( + member.value for member in LiteLLMInternalParam +) + +MCP_INTERNAL_PARAMS: frozenset[str] = frozenset( + { + LiteLLMInternalParam.SKIP_MCP_HANDLER.value, + LiteLLMInternalParam.PRIVATE_SKIP_MCP_HANDLER.value, + LiteLLMInternalParam.MCP_HANDLER_CONTEXT.value, + } +) diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index b67ea91bb0b..05b696b61b9 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -4,9 +4,15 @@ import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, + filter_internal_params, map_finish_reason, reconstruct_model_name, redact_nested_match_and_regex_keys, + strip_internal_params_from_request_body, +) +from litellm.types.internal_params import ( + LITELLM_INTERNAL_REQUEST_BODY_PARAMS, + LiteLLMInternalParam, ) @@ -176,7 +182,11 @@ class TestRedactNestedMatchAndRegexKeys: { "sensitiveInformationPolicy": { "piiEntities": [ - {"type": "NAME", "match": "secret-name", "action": "BLOCKED"} + { + "type": "NAME", + "match": "secret-name", + "action": "BLOCKED", + } ] }, "wordPolicy": { @@ -187,17 +197,61 @@ class TestRedactNestedMatchAndRegexKeys: "regex": "should-redact-key-named-regex", } out = redact_nested_match_and_regex_keys(payload) - assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ - "match" - ] == "[REDACTED]" + assert ( + out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ + "match" + ] + == "[REDACTED]" + ) assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ( "[REDACTED]" ) assert out["regex"] == "[REDACTED]" - assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][ - 0 - ]["match"] == "secret-name" + assert ( + payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ + "match" + ] + == "secret-name" + ) def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None assert redact_nested_match_and_regex_keys("plain") == "plain" + + +class TestInternalParamFiltering: + """The request-body filter must drop every registry key while keeping real provider params.""" + + def test_strips_every_registry_key(self): + seeded = {param.value: "internal" for param in LiteLLMInternalParam} + seeded.update({"temperature": 0.5, "max_tokens": 10}) + + result = strip_internal_params_from_request_body(seeded) + + assert not (LITELLM_INTERNAL_REQUEST_BODY_PARAMS & result.keys()) + assert result == {"temperature": 0.5, "max_tokens": 10} + + def test_keeps_unknown_provider_native_params(self): + # Native provider params we do not enumerate must pass through (no allowlist over-drop). + result = strip_internal_params_from_request_body( + {"anthropic_beta": "x", "top_k": 3} + ) + assert result == {"anthropic_beta": "x", "top_k": 3} + + def test_non_dict_returns_unchanged(self): + assert strip_internal_params_from_request_body("not-a-dict") == "not-a-dict" + + def test_fallback_filter_keeps_non_mcp_internal_params(self): + # filter_internal_params feeds fallback re-dispatch; it must NOT drop + # cache_control_injection_points / stream_chunk_size the way the body filter does. + kwargs = { + "skip_mcp_handler": True, + "cache_control_injection_points": [{"location": "message"}], + "stream_chunk_size": 5, + "temperature": 0.5, + } + result = filter_internal_params(kwargs) + assert "skip_mcp_handler" not in result + assert result["cache_control_injection_points"] == [{"location": "message"}] + assert result["stream_chunk_size"] == 5 + assert result["temperature"] == 0.5 diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index aff89f02ff2..b351044d3c9 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -14,6 +14,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) +from litellm.types.internal_params import LiteLLMInternalParam @pytest.mark.parametrize( @@ -39,3 +40,34 @@ def test_transform_request_drops_stream_chunk_size(config, model): ) assert "stream_chunk_size" not in json.dumps(request_body) + + +@pytest.mark.parametrize( + "model", + [ + "mistral.mistral-7b-instruct-v0:2", + "cohere.command-text-v14", + "amazon.titan-text-express-v1", + "meta.llama3-8b-instruct-v1:0", + "ai21.j2-ultra-v1", + ], +) +def test_invoke_request_does_not_leak_internal_params(model): + """Regression for #30371: the invoke path splats inference_params into the + request body, so internal knobs (e.g. skip_mcp_handler) leaked and strict + Bedrock models rejected the request. Real inference params must survive.""" + seeded = {param.value: "internal" for param in LiteLLMInternalParam} + seeded.update({"max_tokens": 10, "temperature": 0.5}) + + request_body = AmazonInvokeConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=seeded, + litellm_params={}, + headers={}, + ) + + serialized = json.dumps(request_body) + for param in LiteLLMInternalParam: + assert param.value not in serialized, f"{param.value} leaked into {model} body" + assert "max_tokens" in serialized and "temperature" in serialized diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index 9955851132c..fef5f1ae086 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -1004,3 +1004,43 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list( assert "embedding_types" in request_body assert request_body["embedding_types"] == expected_embedding_types assert isinstance(request_body["embedding_types"], list) + + +@pytest.mark.parametrize( + "model", + [ + "bedrock/amazon.titan-embed-image-v1", + "bedrock/amazon.titan-embed-text-v1", + "bedrock/amazon.titan-embed-text-v2:0", + ], +) +def test_bedrock_embedding_does_not_leak_internal_params(model): + """Regression for #30314: cache_control_injection_points (a LiteLLM-internal + knob) leaked into the Titan embeddings body and Bedrock rejected it with + 'extraneous key [cache_control_injection_points] is not permitted'.""" + from litellm.types.internal_params import LiteLLMInternalParam + + client = HTTPHandler() + seeded = {param.value: "internal" for param in LiteLLMInternalParam} + + with patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(titan_embedding_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + litellm.embedding( + model=model, + input=test_input, + client=client, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + api_key="test-bearer-token-12345", + **seeded, + ) + + request_body = json.loads(mock_post.call_args.kwargs.get("data", "{}")) + for param in LiteLLMInternalParam: + assert param.value not in request_body, f"{param.value} leaked into body" + assert request_body.get("inputText") == test_input