diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d4fd0606302..8c59d77aa14 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -891,12 +891,17 @@ class AmazonConverseConfig(BaseConfig): optional_params["maxTokens"] = value if param == "stream": optional_params["stream"] = value - if param == "stop": - if isinstance(value, str): - if len(value) == 0: # converse raises error for empty strings - continue - value = [value] - optional_params["stopSequences"] = value + if param == "stop": + if isinstance(value, str): + if len(value) == 0: # converse raises error for empty strings + continue + value = [value] + elif isinstance(value, list): + # Filter out empty strings from list (Bedrock rejects them) + value = [v for v in value if isinstance(v, str) and len(v) > 0] + if not value: # Skip if empty list or all empty strings + continue + optional_params["stopSequences"] = value if param == "temperature": optional_params["temperature"] = value if param == "top_p": @@ -1433,6 +1438,16 @@ class AmazonConverseConfig(BaseConfig): if tool_choice_values is not None: bedrock_tool_config["toolChoice"] = tool_choice_values + # Extract config block params BEFORE calling _transform_inference_params + # to prevent them from being included in InferenceConfig(**inference_params) + # which would place them inside inferenceConfig instead of at the top level. + # Fixes: serviceTier, guardrailConfig, performanceConfig placement bug (#17336) + config_block_values: dict = {} + for config_name, config_class in self.get_config_blocks().items(): + config_value = inference_params.pop(config_name, None) + if config_value is not None: + config_block_values[config_name] = config_class(**config_value) # type: ignore + data: CommonRequestObject = { "additionalModelRequestFields": additional_request_params, "system": system_content_blocks, @@ -1441,11 +1456,9 @@ class AmazonConverseConfig(BaseConfig): ), } - # Handle all config blocks - for config_name, config_class in self.get_config_blocks().items(): - config_value = inference_params.pop(config_name, None) - if config_value is not None: - data[config_name] = config_class(**config_value) # type: ignore + # Apply config blocks at top-level (already extracted before _transform_inference_params) + for config_name, config_block in config_block_values.items(): + data[config_name] = config_block # type: ignore # Tool Config if bedrock_tool_config is not None: @@ -1633,7 +1646,9 @@ class AmazonConverseConfig(BaseConfig): if "cacheWriteInputTokens" in usage: cache_creation_input_tokens = usage["cacheWriteInputTokens"] input_tokens += cache_creation_input_tokens - + # Recalculate total_tokens to include cache tokens added to input_tokens + # The API's totalTokens doesn't include cache hits/writes, but prompt_tokens does + total_tokens = input_tokens + output_tokens prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cache_read_input_tokens ) diff --git a/tests/litellm/llms/bedrock/test_bedrock_converse_transformation.py b/tests/litellm/llms/bedrock/test_bedrock_converse_transformation.py new file mode 100644 index 00000000000..2659174ebe8 --- /dev/null +++ b/tests/litellm/llms/bedrock/test_bedrock_converse_transformation.py @@ -0,0 +1,66 @@ +import pytest +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.types.llms.bedrock import ConverseTokenUsageBlock + +def test_transform_request_helper_config_blocks(): + config = AmazonConverseConfig() + model = "anthropic.claude-3-haiku-20240307-v1:0" + + # Test serviceTier placement (Bug #17336) + optional_params = { + "serviceTier": {"type": "flex"}, + "guardrailConfig": {"guardrailIdentifier": "test", "guardrailVersion": "1"} + } + + data = config._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params=optional_params + ) + + # Assert top-level placement + assert "serviceTier" in data + assert data["serviceTier"] == {"type": "flex"} + assert "guardrailConfig" in data + + # Assert NOT in inferenceConfig + assert "serviceTier" not in data["inferenceConfig"] + assert "guardrailConfig" not in data["inferenceConfig"] + +def test_transform_usage_recalculation(): + config = AmazonConverseConfig() + usage_block = ConverseTokenUsageBlock( + inputTokens=100, + outputTokens=50, + totalTokens=150, + cacheReadInputTokens=20 + ) + + usage = config._transform_usage(usage_block) + + # Assert input_tokens includes cache tokens + assert usage.prompt_tokens == 120 + # Assert total_tokens is recalculated (Bug 2) + assert usage.total_tokens == 170 + +def test_map_openai_params_stop_filtering(): + config = AmazonConverseConfig() + optional_params = {} + + # Test empty list (Bug 3) + config.map_openai_params( + non_default_params={"stop": []}, + optional_params=optional_params, + model="bedrock/anthropic.claude-3", + drop_params=False + ) + assert "stopSequences" not in optional_params + + # Test list with empty strings + config.map_openai_params( + non_default_params={"stop": ["", "valid"]}, + optional_params=optional_params, + model="bedrock/anthropic.claude-3", + drop_params=False + ) + assert optional_params["stopSequences"] == ["valid"]