From cb2badfebd08050c488753bd483eadb16fe2029b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 19 Jun 2026 04:03:13 +0000 Subject: [PATCH] fix(llm): strip internal params on embedding path and bedrock invoke delegates The request-body filter was wired into the chat path but not into llm_http_handler.embedding(), so providers that splat optional_params into the embedding body (e.g. VoyageAI builds {"input": ..., "model": ..., **optional_params}) still leaked internal knobs and could 400 on a strict schema. Apply strip_internal_params_from_request_body there too. In AmazonInvokeConfig.transform_request the inference_params branches were stripped but the anthropic, nova, twelvelabs and openai delegate branches forwarded the raw optional_params to their sub-transforms. Build one sanitized copy up front and use it for both the inference_params branches and the delegates; this also drops the now-redundant stream_chunk_size pop and the caller-dict mutation. Adds regression coverage for the embedding path and the four invoke delegate providers. --- .../base_invoke_transformation.py | 19 +++--- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- .../test_base_invoke_transformation.py | 32 ++++++++++ .../custom_httpx/test_llm_http_handler.py | 61 +++++++++++++++++++ 4 files changed, 103 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 622df22e3f2..9f6704a8883 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -152,8 +152,10 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): headers: dict, ) -> dict: ## SETUP ## - stream = optional_params.pop("stream", None) - optional_params.pop("stream_chunk_size", None) + sanitized_params = strip_internal_params_from_request_body( + copy.deepcopy(optional_params) + ) + stream = sanitized_params.pop("stream", None) custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {} hf_model_name = litellm_params.get("hf_model_name", None) @@ -165,12 +167,9 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): provider=provider, custom_prompt_dict=custom_prompt_dict, ) - inference_params = strip_internal_params_from_request_body( - copy.deepcopy(optional_params) - ) inference_params = { k: v - for k, v in inference_params.items() + for k, v in sanitized_params.items() if k not in self.aws_authentication_params } request_data: dict = {} @@ -197,7 +196,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): litellm.AmazonAnthropicClaudeConfig().transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=sanitized_params, litellm_params=litellm_params, headers=headers, ) @@ -208,7 +207,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): return litellm.AmazonInvokeNovaConfig().transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=sanitized_params, litellm_params=litellm_params, headers=headers, ) @@ -239,7 +238,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): return litellm.AmazonTwelveLabsPegasusConfig().transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=sanitized_params, litellm_params=litellm_params, headers=headers, ) @@ -248,7 +247,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): return litellm.AmazonBedrockOpenAIConfig().transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params=sanitized_params, litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4aa3202c09f..2e9951159bb 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -895,7 +895,7 @@ class BaseLLMHTTPHandler: data = provider_config.transform_embedding_request( model=model, input=input, - optional_params=optional_params, + optional_params=strip_internal_params_from_request_body(optional_params), headers=headers, ) diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index b351044d3c9..dd8aa8c6728 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -71,3 +71,35 @@ def test_invoke_request_does_not_leak_internal_params(model): for param in LiteLLMInternalParam: assert param.value not in serialized, f"{param.value} leaked into {model} body" assert "max_tokens" in serialized and "temperature" in serialized + + +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-3-sonnet-20240229-v1:0", + "amazon.nova-micro-v1:0", + "twelvelabs.pegasus-1-2-v1:0", + "openai.gpt-oss-20b-1:0", + ], +) +def test_invoke_delegate_paths_do_not_leak_internal_params(model): + """The anthropic, nova, twelvelabs and openai invoke providers delegate to a + sub-transform instead of building the body from inference_params. Those + delegates splat optional_params into their own request body, so the internal + knobs must be stripped before the hand-off or they leak just like the + inference_params splat did (#30371).""" + seeded = {param.value: "internal" for param in LiteLLMInternalParam} + seeded.update({"temperature": 0.5}) + + request_body = AmazonInvokeConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=seeded, + litellm_params={}, + headers={}, + ) + + serialized = json.dumps(request_body) + for param in LiteLLMInternalParam: + assert param.value not in serialized, f"{param.value} leaked into {model} body" + assert "temperature" in serialized diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index f7d445d0788..2b9508687f3 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -18,6 +18,67 @@ from litellm.llms.custom_httpx.llm_http_handler import ( from litellm.types.router import GenericLiteLLMParams +def test_embedding_strips_internal_params_from_request_body(): + """Regression: the embedding path must strip LiteLLM-internal optional_params + before the request body is built. Several embedding transforms (e.g. VoyageAI) + splat optional_params into the wire body, so a leaked internal knob such as + cache_control_injection_points would 400 on a strict-schema provider -- the + same failure the chat path already prevents on line 450.""" + from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig + from litellm.types.internal_params import LiteLLMInternalParam + from litellm.types.utils import EmbeddingResponse + from litellm.utils import ProviderConfigManager + + handler = BaseLLMHTTPHandler() + + captured: dict = {} + + def _capture_post(*args, **kwargs): + captured["data"] = kwargs["data"] + return httpx.Response( + 200, + json={ + "model": "voyage-3", + "object": "list", + "data": [{"embedding": [0.1], "index": 0, "object": "embedding"}], + "usage": {"total_tokens": 1}, + }, + request=httpx.Request("POST", "https://api.voyageai.com/v1/embeddings"), + ) + + mock_client = Mock(spec=HTTPHandler) + mock_client.post = Mock(side_effect=_capture_post) + + seeded = {param.value: "internal" for param in LiteLLMInternalParam} + seeded["output_dimension"] = 256 + + with patch.object( + ProviderConfigManager, + "get_provider_embedding_config", + return_value=VoyageEmbeddingConfig(), + ): + handler.embedding( + model="voyage-3", + input=["hello world"], + timeout=10.0, + custom_llm_provider="voyage", + logging_obj=Mock(), + api_base=None, + optional_params=seeded, + litellm_params={}, + model_response=EmbeddingResponse(), + api_key="test-key", + client=mock_client, + ) + + body = captured["data"] + for param in LiteLLMInternalParam: + assert ( + param.value not in body + ), f"{param.value} leaked into voyage embedding body" + assert "output_dimension" in body + + def test_prepare_fake_stream_request(): # Initialize the BaseLLMHTTPHandler handler = BaseLLMHTTPHandler()