From 37c74f69a8e77006bde063469c83a814092347d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 16:42:15 +0000 Subject: [PATCH 1/8] refactor(bedrock): centralize stream_chunk_size drop via typed control-param set #30240 dropped stream_chunk_size with two hardcoded pops, one per invoke transform entry point. Replace those literals with a single authoritative LiteLLMControlParams type whose field names drive the filter both entry points already apply for aws_authentication_params, so the set of LiteLLM-internal control params lives in one place and the strip happens pre-nesting (covering providers like Titan that nest inference params under textGenerationConfig, which a post-serialization top-level strip would miss). The regression test now asserts on the actual signed wire body that both the base and Anthropic-override transforms funnel through, across four provider/model combinations; all four fail without the filter. --- .../anthropic_claude3_transformation.py | 3 +- .../base_invoke_transformation.py | 3 +- litellm/types/llms/bedrock.py | 13 +++++++++ .../test_base_invoke_transformation.py | 28 +++++++++++++------ 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 79153c3ceff..09e0ec7a44a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.bedrock.common_utils import ( remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import _supports_factory @@ -194,6 +195,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): k: v for k, v in optional_params.items() if k not in self.aws_authentication_params + and k not in LITELLM_CONTROL_PARAM_KEYS } output_config = filtered_params.get("output_config") if isinstance(output_config, dict): @@ -215,7 +217,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) - anthropic_request.pop("stream_chunk_size", None) output_format = anthropic_request.pop("output_format", None) output_config_format = pop_bedrock_invoke_output_config_format( anthropic_request diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 8fc2375c224..36be6818ab8 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, _get_httpx_client, ) +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper @@ -150,7 +151,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): ) -> dict: ## SETUP ## stream = optional_params.pop("stream", None) - optional_params.pop("stream_chunk_size", None) custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {} hf_model_name = litellm_params.get("hf_model_name", None) @@ -167,6 +167,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): k: v for k, v in inference_params.items() if k not in self.aws_authentication_params + and k not in LITELLM_CONTROL_PARAM_KEYS } request_data: dict = {} if provider == "cohere": diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index fa8c3a93ef3..834ec239016 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1,11 +1,24 @@ import json from typing import Any, Dict, List, Literal, Optional, Union +from pydantic import BaseModel from typing_extensions import TYPE_CHECKING, Required, TypedDict, override from .openai import ChatCompletionToolCallChunk +class LiteLLMControlParams(BaseModel): + """LiteLLM-internal control parameters that must never be serialized into a + provider request body. They live in optional_params for convenience but + govern client-side behavior (e.g. how the HTTP response stream is + re-chunked), so Bedrock rejects them as unknown fields.""" + + stream_chunk_size: Optional[int] = None + + +LITELLM_CONTROL_PARAM_KEYS = frozenset(LiteLLMControlParams.model_fields) + + class CachePointBlock(TypedDict, total=False): type: Literal["default"] ttl: str diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index aff89f02ff2..6666d051aea 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,12 +1,9 @@ -import json import os import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../../..")) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, @@ -25,12 +22,15 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation (AmazonAnthropicClaudeConfig, "anthropic.claude-sonnet-4-6"), ], ) -def test_transform_request_drops_stream_chunk_size(config, model): +def test_signed_invoke_body_drops_stream_chunk_size(config, model): """stream_chunk_size is a LiteLLM-internal knob for re-chunking the HTTP response stream. Leaking it into the provider request body makes Bedrock reject the whole request: ValidationException 'stream_chunk_size: Extra - inputs are not permitted'.""" - request_body = config().transform_request( + inputs are not permitted'. The two invoke transform entry points build the + body differently, so this asserts on the actual signed wire bytes that both + funnel through, regardless of which transform produced them.""" + cfg = config() + request_body = cfg.transform_request( model=model, messages=[{"role": "user", "content": "hi"}], optional_params={"stream": True, "stream_chunk_size": 2048, "max_tokens": 10}, @@ -38,4 +38,16 @@ def test_transform_request_drops_stream_chunk_size(config, model): headers={}, ) - assert "stream_chunk_size" not in json.dumps(request_body) + _, signed_body = cfg.sign_request( + headers={}, + optional_params={}, + request_data=request_body, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/{}/invoke".format(model), + api_key="test-bearer-token", + model=model, + stream=True, + ) + + assert signed_body is not None + assert "stream_chunk_size" not in signed_body.decode() + assert "max_tokens" in signed_body.decode() From 4ab7562f86d6007c2d0294c83e6cd687d9f34657 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 16:56:41 +0000 Subject: [PATCH 2/8] style: use union syntax for control-param field to satisfy strict lint --- litellm/types/llms/bedrock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 834ec239016..1212ab20993 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -13,7 +13,7 @@ class LiteLLMControlParams(BaseModel): govern client-side behavior (e.g. how the HTTP response stream is re-chunked), so Bedrock rejects them as unknown fields.""" - stream_chunk_size: Optional[int] = None + stream_chunk_size: int | None = None LITELLM_CONTROL_PARAM_KEYS = frozenset(LiteLLMControlParams.model_fields) From 86fd1358e583a61c1b42bf3f429b8e277ed249f5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:03:59 +0000 Subject: [PATCH 3/8] refactor: consolidate invoke param filtering into one shared method --- .../anthropic_claude3_transformation.py | 8 +------- .../base_invoke_transformation.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 09e0ec7a44a..e84989e4c1a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -24,7 +24,6 @@ from litellm.llms.bedrock.common_utils import ( remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER -from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import _supports_factory @@ -191,12 +190,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: - filtered_params = { - k: v - for k, v in optional_params.items() - if k not in self.aws_authentication_params - and k not in LITELLM_CONTROL_PARAM_KEYS - } + filtered_params = self.filter_invoke_request_params(optional_params) output_config = filtered_params.get("output_config") if isinstance(output_config, dict): filtered_params["output_config"] = dict(output_config) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 36be6818ab8..1c95dd0449d 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -141,6 +141,14 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if k not in inference_params: inference_params[k] = v + def filter_invoke_request_params(self, optional_params: dict) -> dict: + return { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + and k not in LITELLM_CONTROL_PARAM_KEYS + } + def transform_request( self, model: str, @@ -162,13 +170,9 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): provider=provider, custom_prompt_dict=custom_prompt_dict, ) - inference_params = copy.deepcopy(optional_params) - inference_params = { - k: v - for k, v in inference_params.items() - if k not in self.aws_authentication_params - and k not in LITELLM_CONTROL_PARAM_KEYS - } + inference_params = self.filter_invoke_request_params( + copy.deepcopy(optional_params) + ) request_data: dict = {} if provider == "cohere": if model.startswith("cohere.command-r"): From f4b56ae89a23fee7749a3796f054bb1b15625962 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:00:52 +0000 Subject: [PATCH 4/8] refactor(bedrock): type invoke request bodies, split extra_body on drop_params, guard control-param leaks at dispatch Parse each invoke sub-provider's inference params into a typed Pydantic body so known wire keys are kept and unknown keys ride as extra_body passthrough, dropped only when drop_params is set. Add a dispatch-time guard in make_call/make_sync_call (invoke and converse) that aborts the request if a litellm control param such as stream_chunk_size leaked into the body, and point the converse path at the same shared control-key source of truth used by invoke. --- litellm/llms/bedrock/chat/converse_handler.py | 7 +- litellm/llms/bedrock/chat/invoke_handler.py | 3 + .../base_invoke_transformation.py | 9 +- litellm/types/llms/bedrock_invoke.py | 160 ++++++++++++++++++ .../test_base_invoke_transformation.py | 40 ++++- .../llms/bedrock/chat/test_invoke_handler.py | 41 +++++ .../types/llms/test_bedrock_invoke.py | 78 +++++++++ 7 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 litellm/types/llms/bedrock_invoke.py create mode 100644 tests/test_litellm/types/llms/test_bedrock_invoke.py diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 7b1064ccef9..04e82e5ebf0 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -14,6 +14,8 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS +from litellm.types.llms.bedrock_invoke import assert_no_control_params_in_payload from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper @@ -37,6 +39,7 @@ def make_sync_call( if client is None: client = _get_httpx_client() # Create a new client if none provided + assert_no_control_params_in_payload(data) response = client.post( api_base, headers=headers, @@ -268,7 +271,9 @@ class BedrockConverseLLM(BaseAWSLLM): ): ## SETUP ## stream = optional_params.pop("stream", None) - stream_chunk_size = optional_params.pop("stream_chunk_size", None) + stream_chunk_size = optional_params.get("stream_chunk_size") + for _control_key in LITELLM_CONTROL_PARAM_KEYS: + optional_params.pop(_control_key, None) unencoded_model_id = optional_params.pop("model_id", None) fake_stream = optional_params.pop("fake_stream", False) json_mode = optional_params.get("json_mode", False) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 75b560b4d6d..753e41b5343 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -45,6 +45,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.types.llms.bedrock import * +from litellm.types.llms.bedrock_invoke import assert_no_control_params_in_payload from litellm.types.llms.openai import ( ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, @@ -199,6 +200,7 @@ async def make_call( bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, stream_chunk_size: Optional[int] = None, ): + assert_no_control_params_in_payload(data) try: if client is None: client = get_async_httpx_client( @@ -296,6 +298,7 @@ def make_sync_call( bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, stream_chunk_size: Optional[int] = None, ): + assert_no_control_params_in_payload(data) try: if client is None: client = _get_httpx_client( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 1c95dd0449d..a7b4108a734 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -25,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, ) from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS +from litellm.types.llms.bedrock_invoke import parse_invoke_inference_params from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper @@ -170,8 +171,12 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): provider=provider, custom_prompt_dict=custom_prompt_dict, ) - inference_params = self.filter_invoke_request_params( - copy.deepcopy(optional_params) + drop_params = bool(litellm_params.get("drop_params") or litellm.drop_params) + inference_params = parse_invoke_inference_params( + provider=provider, + model=model, + params=self.filter_invoke_request_params(copy.deepcopy(optional_params)), + drop_params=drop_params, ) request_data: dict = {} if provider == "cohere": diff --git a/litellm/types/llms/bedrock_invoke.py b/litellm/types/llms/bedrock_invoke.py new file mode 100644 index 00000000000..60b734777fe --- /dev/null +++ b/litellm/types/llms/bedrock_invoke.py @@ -0,0 +1,160 @@ +"""Typed request bodies for the Bedrock Invoke sub-providers. + +Each sub-provider declares the exact wire keys it accepts in its inference +params. Parsing `optional_params` into one of these models splits the payload +into known fields (kept) and unknown ones captured as `extra_body` passthrough. +The passthrough is forwarded by default and stripped when `drop_params` is set, +so a strict caller never ships keys the provider would reject. +""" + +from __future__ import annotations + +import json +from typing import Dict, List, Optional, Type + +from pydantic import BaseModel, ConfigDict + +from litellm.types.llms.bedrock import LITELLM_CONTROL_PARAM_KEYS + +BEDROCK_INVOKE_PROVIDER = str + + +class BedrockInvokeInferenceParams(BaseModel): + """Base for a sub-provider's inference-param body. + + Declared fields are the provider's wire keys; anything else is captured as + passthrough (`model_extra`) and dropped only when `drop_params` is set. + """ + + model_config = ConfigDict(extra="allow") + + def to_body(self, *, drop_params: bool) -> Dict[str, object]: + known = self.model_dump( + exclude_none=True, exclude=set((self.model_extra or {}).keys()) + ) + if drop_params: + return known + return {**known, **(self.model_extra or {})} + + +class _CohereCommandRBody(BedrockInvokeInferenceParams): + max_tokens: Optional[int] = None + stream: Optional[bool] = None + temperature: Optional[float] = None + p: Optional[float] = None + k: Optional[float] = None + seed: Optional[int] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + stop_sequences: Optional[List[str]] = None + preamble: Optional[str] = None + prompt_truncation: Optional[str] = None + return_prompt: Optional[bool] = None + raw_prompting: Optional[bool] = None + search_queries_only: Optional[bool] = None + + +class _CohereLegacyBody(BedrockInvokeInferenceParams): + max_tokens: Optional[int] = None + stream: Optional[bool] = None + temperature: Optional[float] = None + p: Optional[float] = None + seed: Optional[int] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + stop_sequences: Optional[List[str]] = None + num_generations: Optional[int] = None + return_likelihood: Optional[str] = None + + +class _AI21Body(BedrockInvokeInferenceParams): + maxTokens: Optional[int] = None + temperature: Optional[float] = None + topP: Optional[float] = None + stream: Optional[bool] = None + stopSequences: Optional[List[str]] = None + + +class _MistralBody(BedrockInvokeInferenceParams): + max_tokens: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[float] = None + stream: Optional[bool] = None + stop: Optional[List[str]] = None + + +class _TitanTextGenerationConfig(BedrockInvokeInferenceParams): + maxTokenCount: Optional[int] = None + temperature: Optional[float] = None + topP: Optional[float] = None + stopSequences: Optional[List[str]] = None + + +class _LlamaBody(BedrockInvokeInferenceParams): + max_gen_len: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + topP: Optional[float] = None + stream: Optional[bool] = None + + +_COHERE_COMMAND_R_PREFIX = "cohere.command-r" + +_INVOKE_BODY_MODELS: Dict[str, Type[BedrockInvokeInferenceParams]] = { + "cohere_command_r": _CohereCommandRBody, + "cohere": _CohereLegacyBody, + "ai21": _AI21Body, + "mistral": _MistralBody, + "amazon": _TitanTextGenerationConfig, + "meta": _LlamaBody, + "llama": _LlamaBody, + "deepseek_r1": _LlamaBody, +} + + +def _resolve_invoke_body_model( + provider: Optional[str], model: str +) -> Optional[Type[BedrockInvokeInferenceParams]]: + if provider == "cohere" and model.startswith(_COHERE_COMMAND_R_PREFIX): + return _CohereCommandRBody + if provider is None: + return None + return _INVOKE_BODY_MODELS.get(provider) + + +def parse_invoke_inference_params( + provider: Optional[str], + model: str, + params: Dict[str, object], + drop_params: bool, +) -> Dict[str, object]: + """Validate inference params against the sub-provider's typed body. + + Providers without a typed body (e.g. delegating ones) pass through + unchanged, preserving today's behavior. + """ + body_model = _resolve_invoke_body_model(provider, model) + if body_model is None: + return params + return body_model.model_validate(params).to_body(drop_params=drop_params) + + +def assert_no_control_params(body: Dict[str, object]) -> None: + """Guard run right before dispatch: control keys must never reach the wire.""" + leaked = LITELLM_CONTROL_PARAM_KEYS & body.keys() + if leaked: + raise ValueError( + "litellm control params leaked into the Bedrock request body: " + f"{sorted(leaked)}" + ) + + +def assert_no_control_params_in_payload(data: str) -> None: + """Dispatch-time guard over the serialized request payload.""" + try: + parsed = json.loads(data) + except (TypeError, ValueError): + return + if isinstance(parsed, dict): + assert_no_control_params(parsed) diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index 6666d051aea..96c53591b3c 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -3,7 +3,9 @@ import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) # Adds the parent directory to the system path +sys.path.insert( + 0, os.path.abspath("../../../../../..") +) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, @@ -42,7 +44,9 @@ def test_signed_invoke_body_drops_stream_chunk_size(config, model): headers={}, optional_params={}, request_data=request_body, - api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/{}/invoke".format(model), + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/{}/invoke".format( + model + ), api_key="test-bearer-token", model=model, stream=True, @@ -51,3 +55,35 @@ def test_signed_invoke_body_drops_stream_chunk_size(config, model): assert signed_body is not None assert "stream_chunk_size" not in signed_body.decode() assert "max_tokens" in signed_body.decode() + + +def test_extra_body_passthrough_by_default(): + """Unknown body keys are forwarded verbatim when drop_params is off, so the + soft-allowlist escape hatch keeps working.""" + cfg = AmazonInvokeConfig() + request_body = cfg.transform_request( + model="mistral.mistral-7b-instruct-v0:2", + messages=[{"role": "user", "content": "hi"}], + optional_params={"temperature": 0.5, "made_up_param": "x"}, + litellm_params={}, + headers={}, + ) + + assert request_body["temperature"] == 0.5 + assert request_body["made_up_param"] == "x" + + +def test_drop_params_strips_extra_body_but_keeps_known_params(): + """drop_params selects the strict body: typed provider keys survive, unknown + passthrough keys are dropped instead of being shipped to the provider.""" + cfg = AmazonInvokeConfig() + request_body = cfg.transform_request( + model="mistral.mistral-7b-instruct-v0:2", + messages=[{"role": "user", "content": "hi"}], + optional_params={"temperature": 0.5, "made_up_param": "x"}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert request_body["temperature"] == 0.5 + assert "made_up_param" not in request_body diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index 61987d25d9c..34ce75abe9a 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -297,6 +297,47 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) +@pytest.mark.asyncio +async def test_make_call_guards_against_leaked_control_param(): + """The dispatch wrapper validates the payload right before sending, so a + control param that slipped into the body aborts the request instead of + being rejected downstream by Bedrock.""" + client = MagicMock() + client.post = AsyncMock() + + with pytest.raises(ValueError, match="stream_chunk_size"): + await make_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.command-text-v14/invoke-with-response-stream", + headers={}, + data='{"prompt": "hi", "stream_chunk_size": 2048}', + model="cohere.command-text-v14", + messages=[], + logging_obj=MagicMock(), + ) + + client.post.assert_not_called() + + +def test_make_sync_call_guards_against_leaked_control_param(): + client = MagicMock() + client.post = MagicMock() + + with pytest.raises(ValueError, match="stream_chunk_size"): + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.command-text-v14/invoke-with-response-stream", + headers={}, + data='{"prompt": "hi", "stream_chunk_size": 2048}', + signed_json_body=None, + model="cohere.command-text-v14", + messages=[], + logging_obj=MagicMock(), + ) + + client.post.assert_not_called() + + def test_legacy_bedrock_llm_streaming_does_not_rechunk_by_default(): mock_response = MagicMock() mock_response.status_code = 200 diff --git a/tests/test_litellm/types/llms/test_bedrock_invoke.py b/tests/test_litellm/types/llms/test_bedrock_invoke.py new file mode 100644 index 00000000000..3c5bb33c5be --- /dev/null +++ b/tests/test_litellm/types/llms/test_bedrock_invoke.py @@ -0,0 +1,78 @@ +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.types.llms.bedrock_invoke import ( + assert_no_control_params, + assert_no_control_params_in_payload, + parse_invoke_inference_params, +) + + +def test_passthrough_keeps_unknown_keys(): + body = parse_invoke_inference_params( + provider="mistral", + model="mistral.mistral-7b-instruct-v0:2", + params={"temperature": 0.5, "unknown_key": "x"}, + drop_params=False, + ) + assert body == {"temperature": 0.5, "unknown_key": "x"} + + +def test_drop_params_strips_only_unknown_keys(): + body = parse_invoke_inference_params( + provider="mistral", + model="mistral.mistral-7b-instruct-v0:2", + params={"temperature": 0.5, "unknown_key": "x"}, + drop_params=True, + ) + assert body == {"temperature": 0.5} + + +def test_command_r_and_legacy_resolve_to_different_bodies(): + """command-r exposes k/p; the legacy text model does not, so the same key + must be classified differently per model id.""" + command_r = parse_invoke_inference_params( + provider="cohere", + model="cohere.command-r-v1:0", + params={"k": 2}, + drop_params=True, + ) + legacy = parse_invoke_inference_params( + provider="cohere", + model="cohere.command-text-v14", + params={"k": 2}, + drop_params=True, + ) + assert command_r == {"k": 2.0} + assert legacy == {} + + +def test_unmodeled_provider_passes_through_untouched(): + params = {"anything": 1, "stream_chunk_size": 4} + assert ( + parse_invoke_inference_params( + provider="anthropic", + model="anthropic.claude-sonnet-4-6", + params=params, + drop_params=True, + ) + == params + ) + + +def test_guard_raises_on_leaked_control_param(): + with pytest.raises(ValueError, match="stream_chunk_size"): + assert_no_control_params({"temperature": 0.5, "stream_chunk_size": 2048}) + + +def test_guard_payload_ignores_non_dict_and_invalid_json(): + assert_no_control_params_in_payload("not json") + assert_no_control_params_in_payload("[1, 2, 3]") + assert_no_control_params_in_payload('{"temperature": 0.5}') + + with pytest.raises(ValueError, match="stream_chunk_size"): + assert_no_control_params_in_payload('{"stream_chunk_size": 2048}') From 73ef4bef36c5c74ad5c0b0df8d4d48e68522de8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:07:30 +0000 Subject: [PATCH 5/8] refactor(bedrock): remove dead BedrockLLM invoke path and its tests The Feb 2025 refactor routed Bedrock Invoke through AmazonInvokeConfig and base_llm_http_handler, leaving BedrockLLM.completion and its transitive helpers (process_response, convert_messages_to_prompt, async_completion, async_streaming, _async_anthropic_messages_completion) unreachable, along with AmazonAnthropicClaudeConfig.async_transform_request. Delete them and the unit tests that exercised only that dead path. The class is kept as a thin shell because get_bedrock_invoke_provider is still referenced through it, and the live make_call/make_sync_call dispatch and stream decoders stay put. --- litellm/llms/bedrock/chat/invoke_handler.py | 1038 +---------------- .../anthropic_claude3_transformation.py | 28 - .../test_bedrock_completion.py | 123 -- .../llms/bedrock/chat/test_invoke_handler.py | 34 - 4 files changed, 6 insertions(+), 1217 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 753e41b5343..5772c1edf9a 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -1,19 +1,16 @@ """ -TODO: DELETE FILE. Bedrock LLM is no longer used. Goto `litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py` +Bedrock Invoke streaming dispatch (make_call/make_sync_call) and the event +stream decoders live here. The request/response transforms moved to +`litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py`. """ -import copy -import time import types -from functools import partial from typing import ( AsyncIterator, - Callable, Iterator, Optional, Tuple, cast, - get_args, ) import httpx # type: ignore @@ -25,16 +22,6 @@ from litellm.caching.caching import InMemoryCache from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.logging_utils import track_llm_api_timing -from litellm.litellm_core_utils.prompt_templates.factory import ( - cohere_message_pt, - construct_tool_use_system_prompt, - contains_tag, - custom_prompt, - extract_between_tags, - parse_xml_params, - prompt_factory, -) from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) @@ -65,12 +52,10 @@ from litellm.types.utils import ( StreamingChoices, Usage, ) -from litellm.utils import CustomStreamWrapper, get_secret from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, - ModelResponseIterator, get_bedrock_response_stream_shape, get_bedrock_tool_name, ) @@ -79,9 +64,6 @@ bedrock_tool_name_mappings: InMemoryCache = InMemoryCache( max_size_in_memory=50, default_ttl=600 ) from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( - AmazonBedrockOpenAIConfig, -) converse_config = AmazonConverseConfig() @@ -382,1021 +364,13 @@ def make_sync_call( class BedrockLLM(BaseAWSLLM): - """ - Example call - - ``` - curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \ - --aws-sigv4 "aws:amz:us-east-1:bedrock" \ - --data-raw '{ - "prompt": "Hi", - "temperature": 0, - "p": 0.9, - "max_tokens": 4096 - }' - ``` - """ + """Legacy shell retained only for ``get_bedrock_invoke_provider``, inherited + from BaseAWSLLM and still referenced by name. Invoke requests now run through + AmazonInvokeConfig and base_llm_http_handler.""" def __init__(self) -> None: super().__init__() - @staticmethod - def is_claude_messages_api_model(model: str) -> bool: - """ - Check if the model uses the Claude Messages API (Claude 3+). - - Handles: - - Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-* - - Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-* - - Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4 - """ - # Normalize model string to lowercase for matching - model_lower = model.lower() - - # Claude 3+ indicators (all use Messages API) - messages_api_indicators = [ - "claude-3", # Claude 3.x models - "claude-opus-4", # Claude Opus 4 - "claude-sonnet-4", # Claude Sonnet 4 - "claude-haiku-4", # Claude Haiku 4 - ] - - return any(indicator in model_lower for indicator in messages_api_indicators) - - def convert_messages_to_prompt( - self, model, messages, provider, custom_prompt_dict - ) -> Tuple[str, Optional[list]]: - # handle anthropic prompts and amazon titan prompts - prompt = "" - chat_history: Optional[list] = None - ## CUSTOM PROMPT - if model in custom_prompt_dict: - # check if the model has a registered custom prompt - model_prompt_details = custom_prompt_dict[model] - prompt = custom_prompt( - role_dict=model_prompt_details["roles"], - initial_prompt_value=model_prompt_details.get( - "initial_prompt_value", "" - ), - final_prompt_value=model_prompt_details.get("final_prompt_value", ""), - messages=messages, - ) - return prompt, None - ## ELSE - if provider == "anthropic" or provider == "amazon": - prompt = prompt_factory( - model=model, messages=messages, custom_llm_provider="bedrock" - ) - elif provider == "mistral": - prompt = prompt_factory( - model=model, messages=messages, custom_llm_provider="bedrock" - ) - elif provider == "meta" or provider == "llama": - prompt = prompt_factory( - model=model, messages=messages, custom_llm_provider="bedrock" - ) - elif provider == "openai": - # OpenAI uses messages directly, no prompt conversion needed - # Return empty prompt as it won't be used - prompt = "" - elif provider == "cohere": - prompt, chat_history = cohere_message_pt(messages=messages) - else: - prompt = "" - for message in messages: - if "role" in message: - if message["role"] == "user": - prompt += f"{message['content']}" - else: - prompt += f"{message['content']}" - else: - prompt += f"{message['content']}" - return prompt, chat_history # type: ignore - - def process_response( - self, - model: str, - response: httpx.Response, - model_response: ModelResponse, - stream: Optional[bool], - logging_obj: Logging, - optional_params: dict, - api_key: str, - data: Union[dict, str], - messages: List, - print_verbose, - encoding, - ) -> Union[ModelResponse, CustomStreamWrapper]: - provider = self.get_bedrock_invoke_provider(model) - ## LOGGING - logging_obj.post_call( - input=messages, - api_key=api_key, - original_response=response.text, - additional_args={"complete_input_dict": data}, - ) - print_verbose(f"raw model_response: {response.text}") - - ## RESPONSE OBJECT - try: - completion_response = response.json() - except Exception: - raise BedrockError(message=response.text, status_code=422) - - outputText: Optional[str] = None - try: - if provider == "cohere": - if "text" in completion_response: - outputText = completion_response["text"] # type: ignore - elif "generations" in completion_response: - outputText = completion_response["generations"][0]["text"] - model_response.choices[0].finish_reason = map_finish_reason( - completion_response["generations"][0]["finish_reason"] - ) - elif provider == "anthropic": - if self.is_claude_messages_api_model(model): - json_schemas: dict = {} - _is_function_call = False - ## Handle Tool Calling - if "tools" in optional_params: - _is_function_call = True - for tool in optional_params["tools"]: - json_schemas[tool["function"]["name"]] = tool[ - "function" - ].get("parameters", None) - outputText = completion_response.get("content")[0].get("text", None) - if outputText is not None and contains_tag( - "invoke", outputText - ): # OUTPUT PARSE FUNCTION CALL - function_name = extract_between_tags("tool_name", outputText)[0] - function_arguments_str = extract_between_tags( - "invoke", outputText - )[0].strip() - function_arguments_str = ( - f"{function_arguments_str}" - ) - function_arguments = parse_xml_params( - function_arguments_str, - json_schema=json_schemas.get( - function_name, None - ), # check if we have a json schema for this function name) - ) - _message = litellm.Message( - tool_calls=[ - { - "id": f"call_{uuid.uuid4()}", - "type": "function", - "function": { - "name": function_name, - "arguments": json.dumps(function_arguments), - }, - } - ], - content=None, - ) - model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = ( - outputText # allow user to access raw anthropic tool calling response - ) - if ( - _is_function_call is True - and stream is not None - and stream is True - ): - print_verbose( - "INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK" - ) - # return an iterator - streaming_model_response = ModelResponseStream() - streaming_model_response.choices[0].finish_reason = getattr( - model_response.choices[0], "finish_reason", "stop" - ) - # streaming_model_response.choices = [litellm.utils.StreamingChoices()] - streaming_choice = litellm.utils.StreamingChoices() - streaming_choice.index = model_response.choices[0].index - _tool_calls = [] - print_verbose( - f"type of model_response.choices[0]: {type(model_response.choices[0])}" - ) - print_verbose( - f"type of streaming_choice: {type(streaming_choice)}" - ) - if isinstance(model_response.choices[0], litellm.Choices): - if getattr( - model_response.choices[0].message, "tool_calls", None - ) is not None and isinstance( - model_response.choices[0].message.tool_calls, list - ): - for tool_call in model_response.choices[ - 0 - ].message.tool_calls: - _tool_call = {**tool_call.dict(), "index": 0} - _tool_calls.append(_tool_call) - delta_obj = Delta( - content=getattr( - model_response.choices[0].message, "content", None - ), - role=model_response.choices[0].message.role, - tool_calls=_tool_calls, - ) - streaming_choice.delta = delta_obj - streaming_model_response.choices = [streaming_choice] - completion_stream = ModelResponseIterator( - model_response=streaming_model_response - ) - print_verbose( - "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" - ) - return litellm.CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="cached_response", - logging_obj=logging_obj, - ) - - model_response.choices[0].finish_reason = map_finish_reason( - completion_response.get("stop_reason", "") - ) - _usage = litellm.Usage( - prompt_tokens=completion_response["usage"]["input_tokens"], - completion_tokens=completion_response["usage"]["output_tokens"], - total_tokens=completion_response["usage"]["input_tokens"] - + completion_response["usage"]["output_tokens"], - ) - setattr(model_response, "usage", _usage) - else: - outputText = completion_response["completion"] - - model_response.choices[0].finish_reason = completion_response[ - "stop_reason" - ] - elif provider == "ai21": - outputText = ( - completion_response.get("completions")[0].get("data").get("text") - ) - elif provider == "meta" or provider == "llama": - outputText = completion_response["generation"] - elif provider == "openai": - # OpenAI imported models use OpenAI Chat Completions format - if ( - "choices" in completion_response - and len(completion_response["choices"]) > 0 - ): - choice = completion_response["choices"][0] - if "message" in choice: - outputText = choice["message"].get("content") - elif "text" in choice: # fallback for completion format - outputText = choice["text"] - - # Set finish reason - if "finish_reason" in choice: - model_response.choices[0].finish_reason = map_finish_reason( - choice["finish_reason"] - ) - - # Set usage if available - if "usage" in completion_response: - usage = completion_response["usage"] - _usage = litellm.Usage( - prompt_tokens=usage.get("prompt_tokens", 0), - completion_tokens=usage.get("completion_tokens", 0), - total_tokens=usage.get("total_tokens", 0), - ) - setattr(model_response, "usage", _usage) - elif provider == "mistral": - outputText = completion_response["outputs"][0]["text"] - model_response.choices[0].finish_reason = completion_response[ - "outputs" - ][0]["stop_reason"] - else: # amazon titan - outputText = completion_response.get("results")[0].get("outputText") - except Exception as e: - raise BedrockError( - message="Error processing={}, Received error={}".format( - response.text, str(e) - ), - status_code=422, - ) - - try: - if ( - outputText is not None - and len(outputText) > 0 - and hasattr(model_response.choices[0], "message") - and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore - is None - ): - model_response.choices[0].message.content = outputText # type: ignore - elif ( - hasattr(model_response.choices[0], "message") - and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore - is not None - ): - pass - else: - raise Exception() - except Exception as e: - raise BedrockError( - message="Error parsing received text={}.\nError-{}".format( - outputText, str(e) - ), - status_code=response.status_code, - ) - - if stream and provider == "ai21": - streaming_model_response = ModelResponseStream() - streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore - 0 - ].finish_reason - # streaming_model_response.choices = [litellm.utils.StreamingChoices()] - streaming_choice = litellm.utils.StreamingChoices() - streaming_choice.index = model_response.choices[0].index - delta_obj = litellm.utils.Delta( - content=getattr(model_response.choices[0].message, "content", None), # type: ignore - role=model_response.choices[0].message.role, # type: ignore - ) - streaming_choice.delta = delta_obj - streaming_model_response.choices = [streaming_choice] - mri = ModelResponseIterator(model_response=streaming_model_response) - return CustomStreamWrapper( - completion_stream=mri, - model=model, - custom_llm_provider="cached_response", - logging_obj=logging_obj, - ) - - ## CALCULATING USAGE - bedrock returns usage in the headers - # Skip if usage was already set (e.g., from JSON response for OpenAI provider) - if ( - not hasattr(model_response, "usage") - or getattr(model_response, "usage", None) is None - ): - bedrock_input_tokens = response.headers.get( - "x-amzn-bedrock-input-token-count", None - ) - bedrock_output_tokens = response.headers.get( - "x-amzn-bedrock-output-token-count", None - ) - - prompt_tokens = int( - bedrock_input_tokens or litellm.token_counter(messages=messages) - ) - - completion_tokens = int( - bedrock_output_tokens - or litellm.token_counter( - text=model_response.choices[0].message.content, # type: ignore - count_response_tokens=True, - ) - ) - - model_response.created = int(time.time()) - model_response.model = model - usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - setattr(model_response, "usage", usage) - else: - # Ensure created and model are set even if usage was already set - model_response.created = int(time.time()) - model_response.model = model - - return model_response - - def completion( - self, - model: str, - messages: list, - api_base: Optional[str], - custom_prompt_dict: dict, - model_response: ModelResponse, - print_verbose: Callable, - encoding, - logging_obj: Logging, - optional_params: dict, - acompletion: bool, - timeout: Optional[Union[float, httpx.Timeout]], - litellm_params=None, - logger_fn=None, - extra_headers: Optional[dict] = None, - client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, - ) -> Union[ModelResponse, CustomStreamWrapper]: - try: - from botocore.credentials import Credentials - except ImportError: - raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") - - ## SETUP ## - stream = optional_params.pop("stream", None) - stream_chunk_size = optional_params.pop("stream_chunk_size", None) - - provider = self.get_bedrock_invoke_provider(model) - modelId = self.get_bedrock_model_id( - model=model, - provider=provider, - optional_params=optional_params, - ) - - ## CREDENTIALS ## - # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them - aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) - aws_access_key_id = optional_params.pop("aws_access_key_id", None) - aws_session_token = optional_params.pop("aws_session_token", None) - aws_region_name = optional_params.pop("aws_region_name", None) - aws_role_name = optional_params.pop("aws_role_name", None) - aws_session_name = optional_params.pop("aws_session_name", None) - aws_profile_name = optional_params.pop("aws_profile_name", None) - aws_bedrock_runtime_endpoint = optional_params.pop( - "aws_bedrock_runtime_endpoint", None - ) # https://bedrock-runtime.{region_name}.amazonaws.com - aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) - aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None) - ssl_verify = optional_params.pop("ssl_verify", None) - - ### SET REGION NAME ### - if aws_region_name is None: - # check env # - litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) - - if litellm_aws_region_name is not None and isinstance( - litellm_aws_region_name, str - ): - aws_region_name = litellm_aws_region_name - - standard_aws_region_name = get_secret("AWS_REGION", None) - if standard_aws_region_name is not None and isinstance( - standard_aws_region_name, str - ): - aws_region_name = standard_aws_region_name - - if aws_region_name is None: - aws_region_name = "us-west-2" - - credentials: Credentials = self.get_credentials( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_session_token=aws_session_token, - aws_region_name=aws_region_name, - aws_session_name=aws_session_name, - aws_profile_name=aws_profile_name, - aws_role_name=aws_role_name, - aws_web_identity_token=aws_web_identity_token, - aws_sts_endpoint=aws_sts_endpoint, - ssl_verify=ssl_verify, - ) - - ### SET RUNTIME ENDPOINT ### - endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( - api_base=api_base, - aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, - aws_region_name=aws_region_name, - ) - - if (stream is not None and stream is True) and provider != "ai21": - endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream" - proxy_endpoint_url = ( - f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream" - ) - else: - endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" - - if ( - acompletion - and provider == "anthropic" - and self.is_claude_messages_api_model(model) - ): - if isinstance(client, HTTPHandler): - client = None - return self._async_anthropic_messages_completion( - model=model, - messages=messages, - endpoint_url=endpoint_url, - proxy_endpoint_url=proxy_endpoint_url, - credentials=credentials, - aws_region_name=aws_region_name, - model_response=model_response, - print_verbose=print_verbose, - encoding=encoding, - logging_obj=logging_obj, - optional_params=optional_params, - stream=stream, - litellm_params=litellm_params, - logger_fn=logger_fn, - extra_headers=extra_headers, - timeout=timeout, - client=client, - stream_chunk_size=stream_chunk_size, - ) # type: ignore[return-value] - - prompt, chat_history = self.convert_messages_to_prompt( - model, messages, provider, custom_prompt_dict - ) - inference_params = copy.deepcopy(optional_params) - json_schemas: dict = {} - if provider == "cohere": - if model.startswith("cohere.command-r"): - ## LOAD CONFIG - config = litellm.AmazonCohereChatConfig().get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - _data = {"message": prompt, **inference_params} - if chat_history is not None: - _data["chat_history"] = chat_history - data = json.dumps(_data) - else: - ## LOAD CONFIG - config = litellm.AmazonCohereConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - if stream is True: - inference_params["stream"] = ( - True # cohere requires stream = True in inference params - ) - data = json.dumps({"prompt": prompt, **inference_params}) - elif provider == "anthropic": - if self.is_claude_messages_api_model(model): - # Separate system prompt from rest of message - system_prompt_idx: list[int] = [] - system_messages: list[str] = [] - for idx, message in enumerate(messages): - if message["role"] == "system": - system_messages.append(message["content"]) - system_prompt_idx.append(idx) - if len(system_prompt_idx) > 0: - inference_params["system"] = "\n".join(system_messages) - messages = [ - i for j, i in enumerate(messages) if j not in system_prompt_idx - ] - # Format rest of message according to anthropic guidelines - messages = prompt_factory( - model=model, messages=messages, custom_llm_provider="anthropic_xml" - ) # type: ignore - ## LOAD CONFIG - config = litellm.AmazonAnthropicClaudeConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - ## Handle Tool Calling - if "tools" in inference_params: - _is_function_call = True - for tool in inference_params["tools"]: - json_schemas[tool["function"]["name"]] = tool["function"].get( - "parameters", None - ) - tool_calling_system_prompt = construct_tool_use_system_prompt( - tools=inference_params["tools"] - ) - inference_params["system"] = ( - inference_params.get("system", "\n") - + tool_calling_system_prompt - ) # add the anthropic tool calling prompt to the system prompt - inference_params.pop("tools") - data = json.dumps({"messages": messages, **inference_params}) - else: - ## LOAD CONFIG - config = litellm.AmazonAnthropicConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - data = json.dumps({"prompt": prompt, **inference_params}) - elif provider == "ai21": - ## LOAD CONFIG - config = litellm.AmazonAI21Config.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - - data = json.dumps({"prompt": prompt, **inference_params}) - elif provider == "mistral": - ## LOAD CONFIG - config = litellm.AmazonMistralConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - - data = json.dumps({"prompt": prompt, **inference_params}) - elif provider == "amazon": # amazon titan - ## LOAD CONFIG - config = litellm.AmazonTitanConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - - data = json.dumps( - { - "inputText": prompt, - "textGenerationConfig": inference_params, - } - ) - elif provider == "meta" or provider == "llama": - ## LOAD CONFIG - config = litellm.AmazonLlamaConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - data = json.dumps({"prompt": prompt, **inference_params}) - elif provider == "openai": - ## OpenAI imported models use OpenAI Chat Completions format (messages-based) - # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation - openai_config = AmazonBedrockOpenAIConfig() - supported_params = openai_config.get_supported_openai_params(model=model) - - # Filter to only supported OpenAI params - filtered_params = { - k: v for k, v in inference_params.items() if k in supported_params - } - - # OpenAI uses messages format, not prompt - data = json.dumps({"messages": messages, **filtered_params}) - else: - ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": inference_params, - }, - ) - raise BedrockError( - status_code=404, - message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/`.".format( - provider, model - ), - ) - - ## COMPLETION CALL - - headers = {"Content-Type": "application/json"} - if extra_headers is not None: - headers = {"Content-Type": "application/json", **extra_headers} - prepped = self.get_request_headers( - credentials=credentials, - aws_region_name=aws_region_name, - extra_headers=extra_headers, - endpoint_url=endpoint_url, - data=data, - headers=headers, - ) - - ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": proxy_endpoint_url, - "headers": prepped.headers, - }, - ) - - ### ROUTING (ASYNC, STREAMING, SYNC) - if acompletion: - if isinstance(client, HTTPHandler): - client = None - if stream is True and provider != "ai21": - return self.async_streaming( - model=model, - messages=messages, - data=data, - api_base=proxy_endpoint_url, - model_response=model_response, - print_verbose=print_verbose, - encoding=encoding, - logging_obj=logging_obj, - optional_params=optional_params, - stream=True, - litellm_params=litellm_params, - logger_fn=logger_fn, - headers=prepped.headers, - timeout=timeout, - client=client, - stream_chunk_size=stream_chunk_size, - ) # type: ignore - ### ASYNC COMPLETION - return self.async_completion( - model=model, - messages=messages, - data=data, - api_base=proxy_endpoint_url, - model_response=model_response, - print_verbose=print_verbose, - encoding=encoding, - logging_obj=logging_obj, - optional_params=optional_params, - stream=stream, # type: ignore - litellm_params=litellm_params, - logger_fn=logger_fn, - headers=prepped.headers, - timeout=timeout, - client=client, - ) # type: ignore - - if client is None or isinstance(client, AsyncHTTPHandler): - _params = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - self.client = _get_httpx_client(_params) # type: ignore - else: - self.client = client - if (stream is not None and stream is True) and provider != "ai21": - response = self.client.post( - url=proxy_endpoint_url, - headers=prepped.headers, # type: ignore - data=data, - stream=stream, - logging_obj=logging_obj, - ) - - if response.status_code != 200: - raise BedrockError( - status_code=response.status_code, message=str(response.read()) - ) - - decoder = AWSEventStreamDecoder(model=model) - - completion_stream = decoder.iter_bytes( - response.iter_bytes(chunk_size=stream_chunk_size) - ) - streaming_response = CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="bedrock", - logging_obj=logging_obj, - ) - - ## LOGGING - logging_obj.post_call( - input=messages, - api_key="", - original_response=streaming_response, - additional_args={"complete_input_dict": data}, - ) - return streaming_response - - try: - response = self.client.post( - url=proxy_endpoint_url, - headers=dict(prepped.headers), - data=data, - logging_obj=logging_obj, - ) - response.raise_for_status() - except httpx.HTTPStatusError as err: - error_code = err.response.status_code - raise BedrockError(status_code=error_code, message=err.response.text) - except httpx.TimeoutException: - raise BedrockError(status_code=408, message="Timeout error occurred.") - - return self.process_response( - model=model, - response=response, - model_response=model_response, - stream=stream, - logging_obj=logging_obj, - optional_params=optional_params, - api_key="", - data=data, - messages=messages, - print_verbose=print_verbose, - encoding=encoding, - ) - - async def _async_anthropic_messages_completion( - self, - model: str, - messages: list, - endpoint_url: str, - proxy_endpoint_url: str, - credentials, - aws_region_name: str, - model_response: ModelResponse, - print_verbose: Callable, - encoding, - logging_obj: Logging, - optional_params: dict, - stream, - litellm_params=None, - logger_fn=None, - extra_headers: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - client: Optional[AsyncHTTPHandler] = None, - stream_chunk_size: Optional[int] = None, - ) -> Union[ModelResponse, CustomStreamWrapper]: - transformed_request = ( - await litellm.AmazonAnthropicClaudeConfig().async_transform_request( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params or {}, - headers=extra_headers or {}, - ) - ) - data = json.dumps(transformed_request) - - headers = {"Content-Type": "application/json"} - if extra_headers is not None: - headers = {"Content-Type": "application/json", **extra_headers} - prepped = self.get_request_headers( - credentials=credentials, - aws_region_name=aws_region_name, - extra_headers=extra_headers, - endpoint_url=endpoint_url, - data=data, - headers=headers, - ) - - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": proxy_endpoint_url, - "headers": prepped.headers, - }, - ) - - if stream is True: - return await self.async_streaming( - model=model, - messages=messages, - data=data, - api_base=proxy_endpoint_url, - model_response=model_response, - print_verbose=print_verbose, - encoding=encoding, - logging_obj=logging_obj, - optional_params=optional_params, - stream=True, - litellm_params=litellm_params, - logger_fn=logger_fn, - headers=prepped.headers, - timeout=timeout, - client=client, - stream_chunk_size=stream_chunk_size, - ) - return await self.async_completion( - model=model, - messages=messages, - data=data, - api_base=proxy_endpoint_url, - model_response=model_response, - print_verbose=print_verbose, - encoding=encoding, - logging_obj=logging_obj, - optional_params=optional_params, - stream=stream, # type: ignore - litellm_params=litellm_params, - logger_fn=logger_fn, - headers=prepped.headers, - timeout=timeout, - client=client, - ) - - async def async_completion( - self, - model: str, - messages: list, - api_base: str, - model_response: ModelResponse, - print_verbose: Callable, - data: str, - timeout: Optional[Union[float, httpx.Timeout]], - encoding, - logging_obj: Logging, - stream, - optional_params: dict, - litellm_params=None, - logger_fn=None, - headers={}, - client: Optional[AsyncHTTPHandler] = None, - ) -> Union[ModelResponse, CustomStreamWrapper]: - if client is None: - _params = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = get_async_httpx_client(params=_params, llm_provider=litellm.LlmProviders.BEDROCK) # type: ignore - else: - client = client # type: ignore - - try: - response = await client.post( - api_base, - headers=headers, - data=data, - timeout=timeout, - logging_obj=logging_obj, - ) - response.raise_for_status() - except httpx.HTTPStatusError as err: - error_code = err.response.status_code - raise BedrockError(status_code=error_code, message=err.response.text) - except httpx.TimeoutException: - raise BedrockError(status_code=408, message="Timeout error occurred.") - - return self.process_response( - model=model, - response=response, - model_response=model_response, - stream=stream if isinstance(stream, bool) else False, - logging_obj=logging_obj, - api_key="", - data=data, - messages=messages, - print_verbose=print_verbose, - optional_params=optional_params, - encoding=encoding, - ) - - @track_llm_api_timing() # for streaming, we need to instrument the function calling the wrapper - async def async_streaming( - self, - model: str, - messages: list, - api_base: str, - model_response: ModelResponse, - print_verbose: Callable, - data: str, - timeout: Optional[Union[float, httpx.Timeout]], - encoding, - logging_obj: Logging, - stream, - optional_params: dict, - litellm_params=None, - logger_fn=None, - headers={}, - client: Optional[AsyncHTTPHandler] = None, - stream_chunk_size: Optional[int] = None, - ) -> CustomStreamWrapper: - # The call is not made here; instead, we prepare the necessary objects for the stream. - - streaming_response = CustomStreamWrapper( - completion_stream=None, - make_call=partial( - make_call, - client=client, - api_base=api_base, - headers=headers, - data=data, # type: ignore - model=model, - messages=messages, - logging_obj=logging_obj, - fake_stream=True if "ai21" in api_base else False, - stream_chunk_size=stream_chunk_size, - ), - model=model, - custom_llm_provider="bedrock", - logging_obj=logging_obj, - ) - return streaming_response - - @staticmethod - def _get_provider_from_model_path( - model_path: str, - ) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]: - """ - Helper function to get the provider from a model path with format: provider/model-name - - Args: - model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') - - Returns: - Optional[str]: The provider name, or None if no valid provider found - """ - parts = model_path.split("/") - if len(parts) >= 1: - provider = parts[0] - if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) - return None - class AWSEventStreamDecoder: def __init__(self, model: str, json_mode: Optional[bool] = False) -> None: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index e84989e4c1a..c7b99ddd215 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -154,34 +154,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): return _anthropic_request - async def async_transform_request( - self, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - _anthropic_request = self._build_bedrock_anthropic_request_base( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - headers=headers, - ) - - await self._async_convert_document_url_sources_to_base64(_anthropic_request) - beta_list = self._compute_bedrock_invoke_beta_headers( - model=model, - messages=messages, - optional_params=optional_params, - headers=headers, - ) - if beta_list: - _anthropic_request["anthropic_beta"] = beta_list - - return _anthropic_request - def _build_bedrock_anthropic_request_base( self, model: str, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index fa22ff6b392..655e5284462 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3559,89 +3559,6 @@ def test_bedrock_openai_model_id_extraction(): print(f"✓ Model ID extracted and encoded: {model_id}") -def test_bedrock_openai_convert_messages_to_prompt(): - """ - Test that convert_messages_to_prompt returns empty string for OpenAI models. - """ - from litellm.llms.bedrock.chat.invoke_handler import BedrockLLM - - bedrock_llm = BedrockLLM() - messages = [ - {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "Hello"}, - ] - - prompt, chat_history = bedrock_llm.convert_messages_to_prompt( - model="test-model", messages=messages, provider="openai", custom_prompt_dict={} - ) - - # OpenAI models use messages directly, no prompt conversion - assert prompt == "" - assert chat_history is None - print("✓ convert_messages_to_prompt returns empty for OpenAI") - - -def test_bedrock_openai_response_parsing(): - """ - Test that OpenAI responses are correctly parsed. - """ - from litellm.llms.bedrock.chat.invoke_handler import BedrockLLM - from litellm import ModelResponse - from unittest.mock import Mock - import json - - bedrock_llm = BedrockLLM() - - # Mock OpenAI-style response - openai_response = { - "choices": [ - { - "message": { - "content": "The capital of France is Paris.", - "role": "assistant", - }, - "finish_reason": "stop", - "index": 0, - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, - } - - mock_response = Mock() - mock_response.json.return_value = openai_response - mock_response.text = json.dumps(openai_response) - mock_response.status_code = 200 - mock_response.headers = {} - - model_response = ModelResponse() - mock_logging = Mock() - - result = bedrock_llm.process_response( - model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", - response=mock_response, - model_response=model_response, - stream=False, - logging_obj=mock_logging, - optional_params={}, - api_key="", - data={}, - messages=[{"role": "user", "content": "What is the capital of France?"}], - print_verbose=lambda x: None, - encoding=None, - ) - - # Verify response content - assert result.choices[0].message.content == "The capital of France is Paris." - assert result.choices[0].finish_reason == "stop" - - # Verify usage - assert result.usage.prompt_tokens == 10 - assert result.usage.completion_tokens == 8 - assert result.usage.total_tokens == 18 - - print("✓ OpenAI response parsing works correctly") - - def test_bedrock_openai_request_transformation(): """ Test that the request is correctly transformed for OpenAI models. @@ -3831,46 +3748,6 @@ def test_bedrock_openai_multiple_message_types(): print("✓ Multiple message types handled correctly") -def test_bedrock_openai_error_handling(): - """ - Test that errors from OpenAI models are properly handled. - """ - from litellm.llms.bedrock.chat.invoke_handler import BedrockLLM - from litellm import ModelResponse - from litellm.llms.bedrock.common_utils import BedrockError - from unittest.mock import Mock - import json - - bedrock_llm = BedrockLLM() - - # Mock error response - mock_response = Mock() - mock_response.json.side_effect = Exception("Invalid JSON") - mock_response.text = "Invalid response" - mock_response.status_code = 422 - - model_response = ModelResponse() - mock_logging = Mock() - - with pytest.raises(BedrockError) as exc_info: - bedrock_llm.process_response( - model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", - response=mock_response, - model_response=model_response, - stream=False, - logging_obj=mock_logging, - optional_params={}, - api_key="", - data={}, - messages=[], - print_verbose=lambda x: None, - encoding=None, - ) - - assert exc_info.value.status_code == 422 - print("✓ Error handling works correctly") - - # ============================================================================ # Nova Grounding (web_search_options) Unit Tests (Mocked) # ============================================================================ diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index 34ce75abe9a..9c9d6fc44a3 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -8,14 +8,11 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path -import litellm from litellm.llms.bedrock.chat.invoke_handler import ( AWSEventStreamDecoder, - BedrockLLM, make_call, make_sync_call, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler def test_transform_thinking_blocks_with_redacted_content(): @@ -336,34 +333,3 @@ def test_make_sync_call_guards_against_leaked_control_param(): ) client.post.assert_not_called() - - -def test_legacy_bedrock_llm_streaming_does_not_rechunk_by_default(): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.iter_bytes = MagicMock(return_value=iter([])) - client = HTTPHandler() - client.post = MagicMock(return_value=mock_response) - - BedrockLLM().completion( - model="cohere.command-text-v14", - messages=[{"role": "user", "content": "hi"}], - api_base=None, - custom_prompt_dict={}, - model_response=litellm.ModelResponse(), - print_verbose=lambda *args, **kwargs: None, - encoding=litellm.encoding, - logging_obj=MagicMock(), - optional_params={ - "stream": True, - "aws_access_key_id": "fake", - "aws_secret_access_key": "fake", - "aws_region_name": "us-east-1", - }, - acompletion=False, - timeout=None, - litellm_params={}, - client=client, - ) - - mock_response.iter_bytes.assert_called_once_with(chunk_size=None) From 52d9729a217441e540edecd2bafa36ca97180124 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:13:40 +0000 Subject: [PATCH 6/8] test(bedrock): cover None-provider passthrough in invoke param parsing --- .../test_litellm/types/llms/test_bedrock_invoke.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_litellm/types/llms/test_bedrock_invoke.py b/tests/test_litellm/types/llms/test_bedrock_invoke.py index 3c5bb33c5be..7758d45f599 100644 --- a/tests/test_litellm/types/llms/test_bedrock_invoke.py +++ b/tests/test_litellm/types/llms/test_bedrock_invoke.py @@ -51,6 +51,19 @@ def test_command_r_and_legacy_resolve_to_different_bodies(): assert legacy == {} +def test_none_provider_passes_through_untouched(): + params = {"anything": 1} + assert ( + parse_invoke_inference_params( + provider=None, + model="some-unresolved-model", + params=params, + drop_params=True, + ) + == params + ) + + def test_unmodeled_provider_passes_through_untouched(): params = {"anything": 1, "stream_chunk_size": 4} assert ( From 68986082fa7bb74c003673025d60da961d75638a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:14:41 +0000 Subject: [PATCH 7/8] chore: ignore .coverage artifact --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fda3311fe02..fda5604aa37 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,7 @@ STABILIZATION_TODO.md **/playwright-report **/*.storageState.json **/coverage +.coverage test-config # ---------- Terraform ---------- From dcf2bc7eef77f7247e0cc87154ef2bc3900dbf17 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:16:09 +0000 Subject: [PATCH 8/8] test(bedrock): cover converse dispatch guard against leaked control params --- .../llms/chat/test_converse_handler.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 2a3db5982ef..4995b71d51f 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -202,6 +202,27 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) +def test_make_sync_call_guards_against_leaked_control_param(): + """The converse dispatch wrapper validates the payload right before sending, + so a control param that slipped into the body aborts the request instead of + being rejected downstream by Bedrock.""" + client = MagicMock() + client.post = MagicMock() + + with pytest.raises(ValueError, match="stream_chunk_size"): + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data='{"messages": [], "stream_chunk_size": 2048}', + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + ) + + client.post.assert_not_called() + + def test_completion_plumbs_stream_chunk_size_through_converse(): iter_bytes_spy = _stream_completion_with_spied_iter_bytes( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"