From da86002981b7f867fc4aed2fff6028ea117f23ad Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:16:55 -0800 Subject: [PATCH] fix(bedrock): drop an effort Converse Claude cannot take under drop_params Under drop_params, an effort tier the model cannot take or a malformed effort is dropped with a warning instead of answering 400, and the rest of output_config is still sent. Without drop_params it still gets LiteLLM's 400. An application inference profile ARN with no base_model has its output_config dropped under drop_params, since LiteLLM cannot tell its model, matching Bedrock Invoke. Without drop_params it is still sent. A Claude id the cost map does not list yet counts as taking an effort when the map's fallback rule marks it adaptive thinking, so deployment drop_params no longer drops its effort. --- .../bedrock/chat/converse_transformation.py | 107 ++++++++++++------ .../chat/test_converse_transformation.py | 67 +++++++++++ 2 files changed, 139 insertions(+), 35 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index df1990ddced..7729b3fc7d2 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -142,6 +142,12 @@ ADAPTIVE_THINKING_ON_UNRESOLVED_ARN_WARNING: Final = ( "does not name its model, so LiteLLM cannot tell whether it supports adaptive thinking. Set `base_model` on " "the deployment (for example `us.anthropic.claude-opus-4-8`) to send adaptive thinking as requested." ) +UNRESOLVED_ARN_OUTPUT_CONFIG_WARNING: Final = ( + "Dropping `output_config` for model=%s under drop_params: an application inference profile ARN does not name " + "its model, so LiteLLM cannot tell whether it takes an effort. Set `base_model` on the deployment (for example " + "`us.anthropic.claude-opus-4-8`) to send it." +) +DROP_UNSUPPORTED_EFFORT_WARNING: Final = "Dropping `output_config.effort` for model=%s under drop_params: %s" # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ @@ -576,21 +582,22 @@ class AmazonConverseConfig(BaseConfig): optional_params["output_config"] = existing_output_config optional_params["_output_config_normalized"] = True + @staticmethod + def _anthropic_adaptive_effort_error(model: str, effort: object) -> str | None: + """Why ``output_config.effort`` is not valid on ``model``, or None when it is.""" + valid_efforts: Final = {"high", "medium", "low", "xhigh", "max"} + if not isinstance(effort, str) or effort not in valid_efforts: + return ( + f"Invalid reasoning_effort/output_config.effort value: " + f"{effort!r}. Must be one of: 'low', 'medium', 'high', " + f"'xhigh', or 'max'." + ) + return AnthropicConfig.validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock") + @staticmethod def _validate_anthropic_adaptive_effort(model: str, effort: str) -> None: """Validate ``output_config.effort`` for adaptive-thinking Claude 4.6/4.7.""" - valid_efforts: Final = {"high", "medium", "low", "xhigh", "max"} - if effort not in valid_efforts: - raise litellm.exceptions.BadRequestError( - message=( - f"Invalid reasoning_effort/output_config.effort value: " - f"{effort!r}. Must be one of: 'low', 'medium', 'high', " - f"'xhigh', or 'max'." - ), - model=model, - llm_provider="bedrock_converse", - ) - error = AnthropicConfig.validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock") + error: Final = AmazonConverseConfig._anthropic_adaptive_effort_error(model, effort) if error is not None: raise litellm.exceptions.BadRequestError( message=error, @@ -598,6 +605,46 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock_converse", ) + @staticmethod + def _anthropic_output_config_to_send( + model: str, + base_model: str, + output_config: Mapping[str, object], + already_normalized: bool, + drops_unsupported_effort: bool, + ) -> Mapping[str, object] | None: + if is_bedrock_application_inference_profile_arn(model): + if not drops_unsupported_effort: + return output_config + litellm.verbose_logger.warning(UNRESOLVED_ARN_OUTPUT_CONFIG_WARNING, model) + return None + if not base_model.startswith("anthropic"): + return None + model_takes_effort: Final = AnthropicConfig.model_supports_effort_param( + model, "bedrock" + ) or AnthropicConfig.is_adaptive_thinking_model(model, "bedrock") + if drops_unsupported_effort and not model_takes_effort: + litellm.verbose_logger.warning(DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, model) + return None + normalized_output_config: Final = dict(output_config) + if not already_normalized: + normalize_bedrock_opus_output_config_effort(model=model, output_config=normalized_output_config) + effort: Final = normalized_output_config.get("effort") + effort_error: Final = ( + None if effort is None else AmazonConverseConfig._anthropic_adaptive_effort_error(model, effort) + ) + if effort_error is None: + return normalized_output_config + if not drops_unsupported_effort: + raise litellm.exceptions.BadRequestError( + message=effort_error, + model=model, + llm_provider="bedrock_converse", + ) + litellm.verbose_logger.warning(DROP_UNSUPPORTED_EFFORT_WARNING, model, effort_error) + output_config_without_effort: Final = {k: v for k, v in normalized_output_config.items() if k != "effort"} + return output_config_without_effort or None + @staticmethod def _clamp_thinking_budget_tokens(optional_params: dict) -> None: """ @@ -1765,29 +1812,19 @@ class AmazonConverseConfig(BaseConfig): # This filters: Exception objects, callable objects (functions), Logging objects, etc. additional_request_params = filter_exceptions_from_params(additional_request_params) - if anthropic_output_config is not None and isinstance(anthropic_output_config, dict): - # Application inference profile ARNs hide the underlying model, so the - # effort ceiling and capability gates below cannot run; forward - # verbatim (like ``thinking``) and let Bedrock enforce. - if is_bedrock_application_inference_profile_arn(model): - additional_request_params["output_config"] = anthropic_output_config - elif base_model.startswith("anthropic"): - drops_unsupported_effort: Final = litellm.drop_params is True or drop_params - if drops_unsupported_effort and not AnthropicConfig.model_supports_effort_param(model, "bedrock"): - litellm.verbose_logger.warning( - DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, - model, - ) - else: - if not anthropic_output_config_already_normalized: - normalize_bedrock_opus_output_config_effort( - model=model, - output_config=anthropic_output_config, - ) - effort: Final = anthropic_output_config.get("effort") - if effort is not None: - self._validate_anthropic_adaptive_effort(model=model, effort=effort) - additional_request_params["output_config"] = anthropic_output_config + anthropic_output_config_to_send: Final = ( + self._anthropic_output_config_to_send( + model=model, + base_model=base_model, + output_config=anthropic_output_config, + already_normalized=anthropic_output_config_already_normalized, + drops_unsupported_effort=litellm.drop_params is True or drop_params, + ) + if isinstance(anthropic_output_config, dict) + else None + ) + if anthropic_output_config_to_send is not None: + additional_request_params["output_config"] = anthropic_output_config_to_send return ( inference_params, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index d82ba192b1f..8d809b33020 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5565,6 +5565,73 @@ def test_converse_drop_params_keeps_output_config_for_supporting_anthropic(): assert additional.get("output_config") == {"effort": "high"} +_APPLICATION_INFERENCE_PROFILE_ARN: Final = ( + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456" +) + + +def _converse_output_config(model: str, output_config: dict, litellm_params: dict[str, object]) -> object: + result: Final = AmazonConverseConfig()._transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"maxTokens": 256, "output_config": output_config}, + litellm_params=litellm_params, + headers={}, + ) + return result.get("additionalModelRequestFields", {}).get("output_config") + + +@pytest.mark.parametrize("effort", ["xhigh", "turbo"]) +def test_converse_refuses_an_effort_the_model_cannot_take_without_drop_params(local_model_cost_map, effort): + with pytest.raises(litellm.BadRequestError, match="effort"): + _converse_output_config("us.anthropic.claude-sonnet-4-6", {"effort": effort}, {}) + + +@pytest.mark.parametrize("effort", ["xhigh", "turbo"]) +def test_converse_drops_an_effort_the_model_cannot_take_under_deployment_drop_params( + local_model_cost_map, caplog, effort +): + with caplog.at_level("WARNING"): + sent: Final = _converse_output_config( + "us.anthropic.claude-sonnet-4-6", {"effort": effort}, {"drop_params": True} + ) + + assert sent is None + assert any("output_config.effort" in record.getMessage() for record in caplog.records) + + +def test_converse_forwards_an_unresolved_arn_effort_only_without_drop_params(local_model_cost_map): + assert _converse_output_config(_APPLICATION_INFERENCE_PROFILE_ARN, {"effort": "high"}, {}) == {"effort": "high"} + assert _converse_output_config(_APPLICATION_INFERENCE_PROFILE_ARN, {"effort": "high"}, {"drop_params": True}) is None + + +@pytest.mark.parametrize( + ("base_model", "expected"), + [("us.anthropic.claude-opus-4-7", {"effort": "high"}), ("us.anthropic.claude-haiku-4-5-20251001-v1:0", None)], +) +def test_converse_gates_an_arn_effort_on_its_base_model_under_drop_params(local_model_cost_map, base_model, expected): + litellm_params: Final[dict[str, object]] = {"drop_params": True, "base_model": base_model} + + assert _converse_output_config(_APPLICATION_INFERENCE_PROFILE_ARN, {"effort": "high"}, litellm_params) == expected + + +def test_converse_refuses_a_reasoning_effort_tier_the_model_cannot_take(local_model_cost_map): + with pytest.raises(litellm.BadRequestError, match="xhigh"): + AmazonConverseConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="bedrock/converse/us.anthropic.claude-sonnet-4-6", + drop_params=False, + ) + + +def test_converse_keeps_the_effort_for_a_claude_id_the_cost_map_does_not_list_yet(local_model_cost_map): + model: Final = "us.anthropic.claude-opus-9-9-v1:0" + assert model not in litellm.model_cost + + assert _converse_output_config(model, {"effort": "high"}, {"drop_params": True}) == {"effort": "high"} + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = {