scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge 0fee6acc8b into e0c8a6b483

Browse source
This commit is contained in:
Mateo Wang 2026-06-24 04:56:40 -07:00 • committed by GitHub
commit 4d90f8a4f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 488 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
litellm/litellm_core_utils/core_helpers.py
View file

@ -6,6 +6,11 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union
import httpx
from litellm._logging import verbose_logger
from litellm.types.internal_params import (
LITELLM_CHAT_REQUEST_BODY_STRIP_PARAMS,
LITELLM_INTERNAL_REQUEST_BODY_PARAMS,
MCP_INTERNAL_PARAMS,
)
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
if TYPE_CHECKING:
@ -436,10 +441,11 @@ def filter_internal_params(
data: dict, additional_internal_params: Optional[set] = None
) -> dict:
"""
Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs.
Filter out LiteLLM internal MCP-handler parameters that shouldn't be re-dispatched.
This removes internal/MCP-related parameters that are used by LiteLLM internally
but should not be included in API requests to providers.
Used on completion kwargs (e.g. fallbacks) where the goal is to drop runtime
handler state before re-invoking, not to sanitize a serialized request body.
For the request-body boundary use `strip_internal_params_from_request_body`.
Args:
data: Dictionary of parameters to filter
@ -451,19 +457,46 @@ def filter_internal_params(
if not isinstance(data, dict):
return data
# Known internal parameters that should never be sent to provider APIs
internal_params = {
"skip_mcp_handler",
"mcp_handler_context",
"_skip_mcp_handler",
internal_params = (
MCP_INTERNAL_PARAMS | additional_internal_params
if additional_internal_params
else MCP_INTERNAL_PARAMS
)
return {k: v for k, v in data.items() if k not in internal_params}
def strip_internal_params_from_request_body(data: dict) -> dict:
"""
Remove every LiteLLM-internal optional_params key from a provider request body.
Applied at the serialization boundary (where optional_params becomes a request
body) so internal control knobs can never reach a provider that rejects unknown
fields. See `litellm.types.internal_params.LiteLLMInternalParam` for the registry.
"""
if not isinstance(data, dict):
return data
return {
k: v for k, v in data.items() if k not in LITELLM_INTERNAL_REQUEST_BODY_PARAMS
}
# Add any additional internal params if provided
if additional_internal_params:
internal_params.update(additional_internal_params)
# Filter out internal parameters
return {k: v for k, v in data.items() if k not in internal_params}
def strip_internal_params_from_chat_request_body(data: dict) -> dict:
"""
Strip variant for the chat-completion boundary that preserves keys consumed
inside `transform_request` (currently `cache_control_injection_points`, which
`AmazonConverseConfig` reads to append a `cachePoint` to Bedrock tool_config).
The shared chat handler re-applies `strip_internal_params_from_request_body`
to the body returned by `transform_request`, so splat-style transforms that
splat `**optional_params` into the wire body cannot leak the preserved key.
"""
if not isinstance(data, dict):
return data
return {
k: v for k, v in data.items() if k not in LITELLM_CHAT_REQUEST_BODY_STRIP_PARAMS
}
def redact_nested_match_and_regex_keys(

22
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -8,7 +8,10 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import (
map_finish_reason,
strip_internal_params_from_request_body,
)
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.factory import (
cohere_message_pt,
@ -149,8 +152,10 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
headers: dict,
) -> dict:
## SETUP ##
stream = optional_params.pop("stream", None)
optional_params.pop("stream_chunk_size", None)
sanitized_params = strip_internal_params_from_request_body(
copy.deepcopy(optional_params)
)
stream = sanitized_params.pop("stream", None)
custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {}
hf_model_name = litellm_params.get("hf_model_name", None)
@ -162,10 +167,9 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
provider=provider,
custom_prompt_dict=custom_prompt_dict,
)
inference_params = copy.deepcopy(optional_params)
inference_params = {
k: v
for k, v in inference_params.items()
for k, v in sanitized_params.items()
if k not in self.aws_authentication_params
}
request_data: dict = {}
@ -192,7 +196,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
litellm.AmazonAnthropicClaudeConfig().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=sanitized_params,
litellm_params=litellm_params,
headers=headers,
)
@ -203,7 +207,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
return litellm.AmazonInvokeNovaConfig().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=sanitized_params,
litellm_params=litellm_params,
headers=headers,
)
@ -234,7 +238,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
return litellm.AmazonTwelveLabsPegasusConfig().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=sanitized_params,
litellm_params=litellm_params,
headers=headers,
)
@ -243,7 +247,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
return litellm.AmazonBedrockOpenAIConfig().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=sanitized_params,
litellm_params=litellm_params,
headers=headers,
)

7
litellm/llms/bedrock/embed/embedding.py
View file

@ -11,6 +11,9 @@ import httpx
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_request_body,
)
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -427,7 +430,9 @@ class BedrockEmbedding(BaseAWSLLM):
f"Unable to determine bedrock embedding provider for model: {model}. "
f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
)
inference_params = copy.deepcopy(optional_params)
inference_params = strip_internal_params_from_request_body(
copy.deepcopy(optional_params)
)
inference_params = {
k: v
for k, v in inference_params.items()

8
litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py
View file

@ -14,6 +14,9 @@ from litellm.types.llms.bedrock import (
AmazonNovaCanvasTextToImageRequest,
AmazonNovaCanvasTextToImageResponse,
)
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_request_body,
)
from litellm.llms.bedrock.common_utils import get_cached_model_info
from litellm.types.utils import ImageResponse
@ -73,7 +76,10 @@ class AmazonNovaCanvasConfig:
# Following the same pattern as chat completions and embeddings
unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
image_generation_config = {**image_generation_config, **optional_params}
image_generation_config = {
**image_generation_config,
**strip_internal_params_from_request_body(optional_params),
}
if task_type == "TEXT_IMAGE":
text_to_image_params: Dict[str, Any] = image_generation_config.pop(
"textToImageParams", {}

7
litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py
View file

@ -5,6 +5,9 @@ from typing import List, Optional
from openai.types.image import Image
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_request_body,
)
from litellm.llms.bedrock.common_utils import get_cached_model_info
from litellm.types.utils import ImageResponse
@ -99,7 +102,9 @@ class AmazonStabilityConfig:
text: str,
optional_params: dict,
) -> dict:
inference_params = copy.deepcopy(optional_params)
inference_params = strip_internal_params_from_request_body(
copy.deepcopy(optional_params)
)
inference_params.pop(
"user", None
) # make sure user is not passed in for bedrock call

7
litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py
View file

@ -8,6 +8,9 @@ from litellm.types.llms.bedrock import (
AmazonStability3TextToImageRequest,
AmazonStability3TextToImageResponse,
)
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_request_body,
)
from litellm.llms.bedrock.common_utils import get_cached_model_info
from litellm.types.utils import ImageResponse
@ -73,7 +76,9 @@ class AmazonStability3Config:
"""
Transform the request body for the Stability 3 models
"""
data = AmazonStability3TextToImageRequest(prompt=text, **optional_params)
data = AmazonStability3TextToImageRequest(
prompt=text, **strip_internal_params_from_request_body(optional_params)
)
return data
@classmethod

9
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -8,6 +8,10 @@ import litellm
import litellm.litellm_core_utils
import litellm.types
import litellm.types.utils
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_chat_request_body,
strip_internal_params_from_request_body,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.base_llm.image_variations.transformation import (
BaseImageVariationConfig,
@ -367,10 +371,13 @@ class BaseLLMAIOHTTPHandler:
data = provider_config.transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=strip_internal_params_from_chat_request_body(
optional_params
),
litellm_params=litellm_params,
headers=headers,
)
data = strip_internal_params_from_request_body(data)
## LOGGING
logging_obj.pre_call(

11
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -27,6 +27,10 @@ import litellm.types.utils
from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.core_helpers import (
strip_internal_params_from_chat_request_body,
strip_internal_params_from_request_body,
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@ -490,10 +494,13 @@ class BaseLLMHTTPHandler:
data = provider_config.transform_request(
model=model,
messages=messages,
optional_params=optional_params,
optional_params=strip_internal_params_from_chat_request_body(
optional_params
),
litellm_params=litellm_params,
headers=headers,
)
data = strip_internal_params_from_request_body(data)
if extra_body is not None:
data = {**data, **extra_body}
@ -938,7 +945,7 @@ class BaseLLMHTTPHandler:
data = provider_config.transform_embedding_request(
model=model,
input=input,
optional_params=optional_params,
optional_params=strip_internal_params_from_request_body(optional_params),
headers=headers,
)

44
litellm/types/internal_params.py Normal file
View file

@ -0,0 +1,44 @@
from enum import Enum
class LiteLLMInternalParam(str, Enum):
"""optional_params keys LiteLLM consumes internally and must never serialize into a provider request body.
Strict-schema providers (Bedrock and a growing set of others) reject unknown
fields with a hard 400, so any of these leaking into the wire payload fails
the whole request. This enum is the single source of truth: the request-body
filter derives its key set here, so a newly added internal knob is covered by
adding one member instead of remembering to pop it at every splat site.
"""
SKIP_MCP_HANDLER = "skip_mcp_handler"
PRIVATE_SKIP_MCP_HANDLER = "_skip_mcp_handler"
MCP_HANDLER_CONTEXT = "mcp_handler_context"
STREAM_CHUNK_SIZE = "stream_chunk_size"
FAKE_STREAM = "fake_stream"
CACHE_CONTROL_INJECTION_POINTS = "cache_control_injection_points"
LITELLM_INTERNAL_REQUEST_BODY_PARAMS: frozenset[str] = frozenset(
member.value for member in LiteLLMInternalParam
)
LITELLM_CHAT_REQUEST_BODY_STRIP_PARAMS: frozenset[str] = (
LITELLM_INTERNAL_REQUEST_BODY_PARAMS
- frozenset({LiteLLMInternalParam.CACHE_CONTROL_INJECTION_POINTS.value})
)
"""Variant of `LITELLM_INTERNAL_REQUEST_BODY_PARAMS` for the chat-completion
boundary. `cache_control_injection_points` is consumed inside `transform_request`
by `AmazonConverseConfig` (it appends a `cachePoint` to the Bedrock tool list for
``location: "tool_config"``), so it must reach the transform on the
``converse_like/`` and other shared HTTP handler routes. The shared HTTP handler
re-applies the full strip to the body returned by `transform_request`, so
splat-style transforms cannot leak the preserved key into the wire payload."""
MCP_INTERNAL_PARAMS: frozenset[str] = frozenset(
{
LiteLLMInternalParam.SKIP_MCP_HANDLER.value,
LiteLLMInternalParam.PRIVATE_SKIP_MCP_HANDLER.value,
LiteLLMInternalParam.MCP_HANDLER_CONTEXT.value,
}
)

68
tests/test_litellm/litellm_core_utils/test_core_helpers.py
View file

@ -4,9 +4,15 @@ import pytest
from litellm.litellm_core_utils.core_helpers import (
_FINISH_REASON_MAP,
filter_internal_params,
map_finish_reason,
reconstruct_model_name,
redact_nested_match_and_regex_keys,
strip_internal_params_from_request_body,
)
from litellm.types.internal_params import (
LITELLM_INTERNAL_REQUEST_BODY_PARAMS,
LiteLLMInternalParam,
)
@ -176,7 +182,11 @@ class TestRedactNestedMatchAndRegexKeys:
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}
{
"type": "NAME",
"match": "secret-name",
"action": "BLOCKED",
}
]
},
"wordPolicy": {
@ -187,17 +197,61 @@ class TestRedactNestedMatchAndRegexKeys:
"regex": "should-redact-key-named-regex",
}
out = redact_nested_match_and_regex_keys(payload)
assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
"match"
] == "[REDACTED]"
assert (
out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
"match"
]
== "[REDACTED]"
)
assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == (
"[REDACTED]"
)
assert out["regex"] == "[REDACTED]"
assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][
0
]["match"] == "secret-name"
assert (
payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
"match"
]
== "secret-name"
)
def test_passes_through_none_and_str(self):
assert redact_nested_match_and_regex_keys(None) is None
assert redact_nested_match_and_regex_keys("plain") == "plain"
class TestInternalParamFiltering:
"""The request-body filter must drop every registry key while keeping real provider params."""
def test_strips_every_registry_key(self):
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded.update({"temperature": 0.5, "max_tokens": 10})
result = strip_internal_params_from_request_body(seeded)
assert not (LITELLM_INTERNAL_REQUEST_BODY_PARAMS & result.keys())
assert result == {"temperature": 0.5, "max_tokens": 10}
def test_keeps_unknown_provider_native_params(self):
# Native provider params we do not enumerate must pass through (no allowlist over-drop).
result = strip_internal_params_from_request_body(
{"anthropic_beta": "x", "top_k": 3}
)
assert result == {"anthropic_beta": "x", "top_k": 3}
def test_non_dict_returns_unchanged(self):
assert strip_internal_params_from_request_body("not-a-dict") == "not-a-dict"
def test_fallback_filter_keeps_non_mcp_internal_params(self):
# filter_internal_params feeds fallback re-dispatch; it must NOT drop
# cache_control_injection_points / stream_chunk_size the way the body filter does.
kwargs = {
"skip_mcp_handler": True,
"cache_control_injection_points": [{"location": "message"}],
"stream_chunk_size": 5,
"temperature": 0.5,
}
result = filter_internal_params(kwargs)
assert "skip_mcp_handler" not in result
assert result["cache_control_injection_points"] == [{"location": "message"}]
assert result["stream_chunk_size"] == 5
assert result["temperature"] == 0.5

64
tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py
View file

@ -14,6 +14,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from litellm.types.internal_params import LiteLLMInternalParam
@pytest.mark.parametrize(
@ -39,3 +40,66 @@ def test_transform_request_drops_stream_chunk_size(config, model):
)
assert "stream_chunk_size" not in json.dumps(request_body)
@pytest.mark.parametrize(
"model",
[
"mistral.mistral-7b-instruct-v0:2",
"cohere.command-text-v14",
"amazon.titan-text-express-v1",
"meta.llama3-8b-instruct-v1:0",
"ai21.j2-ultra-v1",
],
)
def test_invoke_request_does_not_leak_internal_params(model):
"""Regression for #30371: the invoke path splats inference_params into the
request body, so internal knobs (e.g. skip_mcp_handler) leaked and strict
Bedrock models rejected the request. Real inference params must survive."""
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded.update({"max_tokens": 10, "temperature": 0.5})
request_body = AmazonInvokeConfig().transform_request(
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params=seeded,
litellm_params={},
headers={},
)
serialized = json.dumps(request_body)
for param in LiteLLMInternalParam:
assert param.value not in serialized, f"{param.value} leaked into {model} body"
assert "max_tokens" in serialized and "temperature" in serialized
@pytest.mark.parametrize(
"model",
[
"anthropic.claude-3-sonnet-20240229-v1:0",
"amazon.nova-micro-v1:0",
"twelvelabs.pegasus-1-2-v1:0",
"openai.gpt-oss-20b-1:0",
],
)
def test_invoke_delegate_paths_do_not_leak_internal_params(model):
"""The anthropic, nova, twelvelabs and openai invoke providers delegate to a
sub-transform instead of building the body from inference_params. Those
delegates splat optional_params into their own request body, so the internal
knobs must be stripped before the hand-off or they leak just like the
inference_params splat did (#30371)."""
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded.update({"temperature": 0.5})
request_body = AmazonInvokeConfig().transform_request(
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params=seeded,
litellm_params={},
headers={},
)
serialized = json.dumps(request_body)
for param in LiteLLMInternalParam:
assert param.value not in serialized, f"{param.value} leaked into {model} body"
assert "temperature" in serialized

40
tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py
View file

@ -1004,3 +1004,43 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list(
assert "embedding_types" in request_body
assert request_body["embedding_types"] == expected_embedding_types
assert isinstance(request_body["embedding_types"], list)
@pytest.mark.parametrize(
"model",
[
"bedrock/amazon.titan-embed-image-v1",
"bedrock/amazon.titan-embed-text-v1",
"bedrock/amazon.titan-embed-text-v2:0",
],
)
def test_bedrock_embedding_does_not_leak_internal_params(model):
"""Regression for #30314: cache_control_injection_points (a LiteLLM-internal
knob) leaked into the Titan embeddings body and Bedrock rejected it with
'extraneous key [cache_control_injection_points] is not permitted'."""
from litellm.types.internal_params import LiteLLMInternalParam
client = HTTPHandler()
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.text = json.dumps(titan_embedding_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response
litellm.embedding(
model=model,
input=test_input,
client=client,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com",
api_key="test-bearer-token-12345",
**seeded,
)
request_body = json.loads(mock_post.call_args.kwargs.get("data", "{}"))
for param in LiteLLMInternalParam:
assert param.value not in request_body, f"{param.value} leaked into body"
assert request_body.get("inputText") == test_input

178
tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
View file

@ -26,6 +26,184 @@ _ACTIVE_KEY = "_code_interpreter_interception_active"
_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
def test_embedding_strips_internal_params_from_request_body():
"""Regression: the embedding path must strip LiteLLM-internal optional_params
before the request body is built. Several embedding transforms (e.g. VoyageAI)
splat optional_params into the wire body, so a leaked internal knob such as
cache_control_injection_points would 400 on a strict-schema provider -- the
same failure the chat path already prevents on line 450."""
from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig
from litellm.types.internal_params import LiteLLMInternalParam
from litellm.types.utils import EmbeddingResponse
from litellm.utils import ProviderConfigManager
handler = BaseLLMHTTPHandler()
captured: dict = {}
def _capture_post(*args, **kwargs):
captured["data"] = kwargs["data"]
return httpx.Response(
200,
json={
"model": "voyage-3",
"object": "list",
"data": [{"embedding": [0.1], "index": 0, "object": "embedding"}],
"usage": {"total_tokens": 1},
},
request=httpx.Request("POST", "https://api.voyageai.com/v1/embeddings"),
)
mock_client = Mock(spec=HTTPHandler)
mock_client.post = Mock(side_effect=_capture_post)
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded["output_dimension"] = 256
with patch.object(
ProviderConfigManager,
"get_provider_embedding_config",
return_value=VoyageEmbeddingConfig(),
):
handler.embedding(
model="voyage-3",
input=["hello world"],
timeout=10.0,
custom_llm_provider="voyage",
logging_obj=Mock(),
api_base=None,
optional_params=seeded,
litellm_params={},
model_response=EmbeddingResponse(),
api_key="test-key",
client=mock_client,
)
body = captured["data"]
for param in LiteLLMInternalParam:
assert (
param.value not in body
), f"{param.value} leaked into voyage embedding body"
assert "output_dimension" in body
def test_chat_boundary_preserves_cache_control_injection_points():
"""Regression: the chat-completion boundary must NOT strip
`cache_control_injection_points`. AmazonConverseConfig.transform_request
consumes that key to append a `cachePoint` to Bedrock tool_config (used by
the `converse_like/` route, which goes through this shared handler), so
stripping it here silently disables tool-config prompt caching. Universal
LiteLLM-internal knobs (skip_mcp_handler, fake_stream, ...) must still be
stripped before the transform splats optional_params into the wire body."""
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.internal_params import LiteLLMInternalParam
from litellm.types.utils import ModelResponse
handler = BaseLLMHTTPHandler()
captured: dict = {}
provider_config = Mock()
provider_config.should_fake_stream.return_value = False
provider_config.validate_environment.return_value = {}
provider_config.get_complete_url.return_value = "https://example.invalid/chat"
provider_config.sign_request.return_value = ({}, None)
def _capture_transform_request(*, model, messages, optional_params, **_):
captured["optional_params"] = optional_params
raise RuntimeError("stop after capture")
provider_config.transform_request.side_effect = _capture_transform_request
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded["cache_control_injection_points"] = [{"location": "tool_config"}]
seeded["temperature"] = 0.5
with pytest.raises(RuntimeError, match="stop after capture"):
handler.completion(
model="anthropic.claude-3-5-haiku-20241022-v1:0",
messages=[{"role": "user", "content": "hi"}],
api_base=None,
custom_llm_provider="bedrock",
model_response=ModelResponse(),
encoding=None,
logging_obj=Mock(),
optional_params=seeded,
timeout=10.0,
litellm_params={},
acompletion=False,
provider_config=provider_config,
)
forwarded = captured["optional_params"]
assert forwarded["cache_control_injection_points"] == [{"location": "tool_config"}]
for param in LiteLLMInternalParam:
if param is LiteLLMInternalParam.CACHE_CONTROL_INJECTION_POINTS:
continue
assert (
param.value not in forwarded
), f"{param.value} leaked past the chat-completion strip"
def test_chat_boundary_strips_internal_params_from_splat_body():
"""Regression: `cache_control_injection_points` is preserved in `optional_params`
so AmazonConverseConfig can consume it, but splat-style transforms (OpenAI,
Anthropic, OpenAI-compatible) build the wire body with `**optional_params` and
never pop it. The shared handler must therefore strip internal params from the
body returned by `transform_request` to prevent extraneous-field 400s on
strict-schema providers."""
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.internal_params import LiteLLMInternalParam
from litellm.types.utils import ModelResponse
handler = BaseLLMHTTPHandler()
captured: dict = {}
provider_config = Mock()
provider_config.should_fake_stream.return_value = False
provider_config.validate_environment.return_value = {}
provider_config.get_complete_url.return_value = "https://example.invalid/chat"
def _splat_transform_request(*, model, messages, optional_params, **_):
return {"model": model, "messages": messages, **optional_params}
provider_config.transform_request.side_effect = _splat_transform_request
def _capture_sign_request(*, request_data, headers, **_):
captured["body"] = request_data
raise RuntimeError("stop after capture")
provider_config.sign_request.side_effect = _capture_sign_request
seeded = {param.value: "internal" for param in LiteLLMInternalParam}
seeded["cache_control_injection_points"] = [{"location": "tool_config"}]
seeded["temperature"] = 0.5
with pytest.raises(RuntimeError, match="stop after capture"):
handler.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
api_base=None,
custom_llm_provider="openai",
model_response=ModelResponse(),
encoding=None,
logging_obj=Mock(),
optional_params=seeded,
timeout=10.0,
litellm_params={},
acompletion=False,
provider_config=provider_config,
)
body = captured["body"]
for param in LiteLLMInternalParam:
assert (
param.value not in body
), f"{param.value} leaked into the wire body past the splat transform"
assert body["temperature"] == 0.5
def test_prepare_fake_stream_request():
# Initialize the BaseLLMHTTPHandler
handler = BaseLLMHTTPHandler()