mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
fix(llm): strip internal params from chat body after transform_request
The chat-completion boundary preserves cache_control_injection_points in optional_params so AmazonConverseConfig.transform_request can pop it (used for Bedrock tool_config cachePoint injection on the converse_like/ route). However, splat-style transforms such as OpenAIGPTConfig and AnthropicConfig build the wire body with ```**optional_params``` and never pop that key, so the preserved key leaked straight into the wire payload and re-introduced the extraneous-field 400s on strict-schema providers. Apply strip_internal_params_from_request_body to the dict returned by transform_request inside the shared HTTP handler. For Converse the key was already popped, so the post-transform strip is a no-op; for splat transforms it closes the leak at the serialization boundary without touching every provider's transform_request.
This commit is contained in:
parent
8a14dc0dda
commit
0fee6acc8b
5 changed files with 68 additions and 4 deletions
|
|
@ -487,8 +487,9 @@ def strip_internal_params_from_chat_request_body(data: dict) -> dict:
|
|||
Strip variant for the chat-completion boundary that preserves keys consumed
|
||||
inside `transform_request` (currently `cache_control_injection_points`, which
|
||||
`AmazonConverseConfig` reads to append a `cachePoint` to Bedrock tool_config).
|
||||
Splat-style transforms still call `strip_internal_params_from_request_body`
|
||||
themselves before serialization, so the param never leaks into the wire body.
|
||||
The shared chat handler re-applies `strip_internal_params_from_request_body`
|
||||
to the body returned by `transform_request`, so splat-style transforms that
|
||||
splat `**optional_params` into the wire body cannot leak the preserved key.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import litellm.types
|
|||
import litellm.types.utils
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
strip_internal_params_from_chat_request_body,
|
||||
strip_internal_params_from_request_body,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
from litellm.llms.base_llm.image_variations.transformation import (
|
||||
|
|
@ -376,6 +377,7 @@ class BaseLLMAIOHTTPHandler:
|
|||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
data = strip_internal_params_from_request_body(data)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
|
|
|
|||
|
|
@ -454,6 +454,7 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
data = strip_internal_params_from_request_body(data)
|
||||
|
||||
if extra_body is not None:
|
||||
data = {**data, **extra_body}
|
||||
|
|
|
|||
|
|
@ -31,8 +31,9 @@ LITELLM_CHAT_REQUEST_BODY_STRIP_PARAMS: frozenset[str] = (
|
|||
boundary. `cache_control_injection_points` is consumed inside `transform_request`
|
||||
by `AmazonConverseConfig` (it appends a `cachePoint` to the Bedrock tool list for
|
||||
``location: "tool_config"``), so it must reach the transform on the
|
||||
``converse_like/`` and other shared HTTP handler routes. Splat-style transforms
|
||||
that never consume it strip the full set themselves before serialization."""
|
||||
``converse_like/`` and other shared HTTP handler routes. The shared HTTP handler
|
||||
re-applies the full strip to the body returned by `transform_request`, so
|
||||
splat-style transforms cannot leak the preserved key into the wire payload."""
|
||||
|
||||
MCP_INTERNAL_PARAMS: frozenset[str] = frozenset(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -137,6 +137,65 @@ def test_chat_boundary_preserves_cache_control_injection_points():
|
|||
), f"{param.value} leaked past the chat-completion strip"
|
||||
|
||||
|
||||
def test_chat_boundary_strips_internal_params_from_splat_body():
|
||||
"""Regression: `cache_control_injection_points` is preserved in `optional_params`
|
||||
so AmazonConverseConfig can consume it, but splat-style transforms (OpenAI,
|
||||
Anthropic, OpenAI-compatible) build the wire body with `**optional_params` and
|
||||
never pop it. The shared handler must therefore strip internal params from the
|
||||
body returned by `transform_request` to prevent extraneous-field 400s on
|
||||
strict-schema providers."""
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.types.internal_params import LiteLLMInternalParam
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
provider_config = Mock()
|
||||
provider_config.should_fake_stream.return_value = False
|
||||
provider_config.validate_environment.return_value = {}
|
||||
provider_config.get_complete_url.return_value = "https://example.invalid/chat"
|
||||
|
||||
def _splat_transform_request(*, model, messages, optional_params, **_):
|
||||
return {"model": model, "messages": messages, **optional_params}
|
||||
|
||||
provider_config.transform_request.side_effect = _splat_transform_request
|
||||
|
||||
def _capture_sign_request(*, request_data, headers, **_):
|
||||
captured["body"] = request_data
|
||||
raise RuntimeError("stop after capture")
|
||||
|
||||
provider_config.sign_request.side_effect = _capture_sign_request
|
||||
|
||||
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
|
||||
seeded["cache_control_injection_points"] = [{"location": "tool_config"}]
|
||||
seeded["temperature"] = 0.5
|
||||
|
||||
with pytest.raises(RuntimeError, match="stop after capture"):
|
||||
handler.completion(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=None,
|
||||
custom_llm_provider="openai",
|
||||
model_response=ModelResponse(),
|
||||
encoding=None,
|
||||
logging_obj=Mock(),
|
||||
optional_params=seeded,
|
||||
timeout=10.0,
|
||||
litellm_params={},
|
||||
acompletion=False,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
body = captured["body"]
|
||||
for param in LiteLLMInternalParam:
|
||||
assert (
|
||||
param.value not in body
|
||||
), f"{param.value} leaked into the wire body past the splat transform"
|
||||
assert body["temperature"] == 0.5
|
||||
|
||||
|
||||
def test_prepare_fake_stream_request():
|
||||
# Initialize the BaseLLMHTTPHandler
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue