diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 79153c3ceff..09e0ec7a44a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.bedrock.common_utils import ( remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import _supports_factory @@ -194,6 +195,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): k: v for k, v in optional_params.items() if k not in self.aws_authentication_params + and k not in LITELLM_CONTROL_PARAM_KEYS } output_config = filtered_params.get("output_config") if isinstance(output_config, dict): @@ -215,7 +217,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) - anthropic_request.pop("stream_chunk_size", None) output_format = anthropic_request.pop("output_format", None) output_config_format = pop_bedrock_invoke_output_config_format( anthropic_request diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 8fc2375c224..36be6818ab8 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, _get_httpx_client, ) +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper @@ -150,7 +151,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): ) -> dict: ## SETUP ## stream = optional_params.pop("stream", None) - optional_params.pop("stream_chunk_size", None) custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {} hf_model_name = litellm_params.get("hf_model_name", None) @@ -167,6 +167,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): k: v for k, v in inference_params.items() if k not in self.aws_authentication_params + and k not in LITELLM_CONTROL_PARAM_KEYS } request_data: dict = {} if provider == "cohere": diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index fa8c3a93ef3..834ec239016 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1,11 +1,24 @@ import json from typing import Any, Dict, List, Literal, Optional, Union +from pydantic import BaseModel from typing_extensions import TYPE_CHECKING, Required, TypedDict, override from .openai import ChatCompletionToolCallChunk +class LiteLLMControlParams(BaseModel): + """LiteLLM-internal control parameters that must never be serialized into a + provider request body. They live in optional_params for convenience but + govern client-side behavior (e.g. how the HTTP response stream is + re-chunked), so Bedrock rejects them as unknown fields.""" + + stream_chunk_size: Optional[int] = None + + +LITELLM_CONTROL_PARAM_KEYS = frozenset(LiteLLMControlParams.model_fields) + + class CachePointBlock(TypedDict, total=False): type: Literal["default"] ttl: str diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index aff89f02ff2..6666d051aea 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,12 +1,9 @@ -import json import os import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../../..")) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, @@ -25,12 +22,15 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation (AmazonAnthropicClaudeConfig, "anthropic.claude-sonnet-4-6"), ], ) -def test_transform_request_drops_stream_chunk_size(config, model): +def test_signed_invoke_body_drops_stream_chunk_size(config, model): """stream_chunk_size is a LiteLLM-internal knob for re-chunking the HTTP response stream. Leaking it into the provider request body makes Bedrock reject the whole request: ValidationException 'stream_chunk_size: Extra - inputs are not permitted'.""" - request_body = config().transform_request( + inputs are not permitted'. The two invoke transform entry points build the + body differently, so this asserts on the actual signed wire bytes that both + funnel through, regardless of which transform produced them.""" + cfg = config() + request_body = cfg.transform_request( model=model, messages=[{"role": "user", "content": "hi"}], optional_params={"stream": True, "stream_chunk_size": 2048, "max_tokens": 10}, @@ -38,4 +38,16 @@ def test_transform_request_drops_stream_chunk_size(config, model): headers={}, ) - assert "stream_chunk_size" not in json.dumps(request_body) + _, signed_body = cfg.sign_request( + headers={}, + optional_params={}, + request_data=request_body, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/{}/invoke".format(model), + api_key="test-bearer-token", + model=model, + stream=True, + ) + + assert signed_body is not None + assert "stream_chunk_size" not in signed_body.decode() + assert "max_tokens" in signed_body.decode()