diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e1387a9068c..a3c76d6a29b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1992,19 +1992,35 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return data def _apply_output_config(self, data: dict, model: str, optional_params: dict) -> None: - """Validate and apply output_config to the request data.""" + """Validate and apply output_config to the request data. + + The ``drop_params`` gate here is an effort gate: ``format`` is a + structured-output field, not an effort field, so it survives the drop + and is vetted where it is consumed (the map's + ``supports_native_structured_output`` flag on emission paths). + """ if "output_config" not in optional_params: return output_config: Final = optional_params.get("output_config") if not output_config or not isinstance(output_config, dict): return - if litellm.drop_params is True and not self._model_supports_effort_param(model, self._resolved_provider): + if ( + litellm.drop_params is True + and any(key != "format" for key in output_config) + and not self._model_supports_effort_param(model, self._resolved_provider) + ): litellm.verbose_logger.warning( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, model, ) - optional_params.pop("output_config", None) - data.pop("output_config", None) + preserved_format: Final = output_config.get("format") + if preserved_format is None: + optional_params.pop("output_config", None) + data.pop("output_config", None) + return + format_only: Final = {"format": preserved_format} # mutable-ok: json body + optional_params["output_config"] = format_only # rebind-ok: out-param store + data["output_config"] = format_only # rebind-ok: out-param store return effort: Final = output_config.get("effort") valid_efforts: Final = ["high", "medium", "low", "xhigh", "max"] diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 40b90014f3b..8e709349400 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers -from litellm.litellm_core_utils.litellm_logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -16,17 +15,16 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( - convert_bedrock_invoke_output_format_to_inline_schema, + apply_bedrock_invoke_structured_output, get_anthropic_beta_from_headers, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, - pop_bedrock_invoke_output_config_format, + strip_unsupported_bedrock_invoke_output_config_keys, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse -from litellm.utils import _supports_factory if TYPE_CHECKING: import tiktoken @@ -212,36 +210,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) anthropic_request.pop("stream_chunk_size", None) - output_format: Final = anthropic_request.pop("output_format", None) - output_config_format: Final = pop_bedrock_invoke_output_config_format(anthropic_request) - if output_format: - convert_bedrock_invoke_output_format_to_inline_schema( - output_format=output_format, - request_body=anthropic_request, - ) - elif output_config_format: - convert_bedrock_invoke_output_format_to_inline_schema( - output_format=output_config_format, - request_body=anthropic_request, - ) - if not ( - _supports_factory( - model=model, - custom_llm_provider="bedrock", - key="supports_output_config", - ) - or AnthropicConfig._model_supports_effort_param(model, "bedrock") - ): - if anthropic_request.pop("output_config", None) is not None: - verbose_logger.warning( - "Bedrock Invoke: stripping unsupported `output_config` for " - "model=%s — neither `supports_output_config` nor any " - "`supports_*_reasoning_effort` flag is set in " - "model_prices_and_context_window.json. Add the capability " - "flag to the model JSON entry if this model accepts " - "`output_config`.", - model, - ) + apply_bedrock_invoke_structured_output( + model=model, + request_body=anthropic_request, + ) + strip_unsupported_bedrock_invoke_output_config_keys( + model=model, + request_body=anthropic_request, + ) if "anthropic_version" not in anthropic_request: anthropic_request["anthropic_version"] = self.anthropic_version diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9cbceb4880c..30a77d57f24 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -177,6 +177,95 @@ def convert_bedrock_invoke_output_format_to_inline_schema( request_body["messages"] = new_messages +def _bedrock_model_supports(model: str, key: str) -> bool: + from litellm.utils import _supports_factory + + return _supports_factory(model=model, custom_llm_provider="bedrock", key=key) + + +def apply_bedrock_invoke_structured_output( + model: str, + request_body: dict[str, object], # mutable-ok: edited in place like siblings +) -> None: + """ + Route Anthropic structured-output params to what the Bedrock model supports. + + Consumes the legacy top-level ``output_format`` and the newer + ``output_config.format``, keeping the pre-existing precedence of the legacy + field when a request carries both. Models flagged + ``supports_native_structured_output`` in the model map get the schema + forwarded as ``output_config.format``, which Bedrock relays to the model for + enforced structured output. For every other model the schema is inlined into + the last user message as best-effort text, with a warning because nothing + enforces it. + """ + legacy_output_format: Final = request_body.pop("output_format", None) + output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body) + schema_format: Final = legacy_output_format if isinstance(legacy_output_format, dict) else output_config_format + if schema_format is None: + return + + if _bedrock_model_supports(model, "supports_native_structured_output"): + existing_output_config: Final = request_body.get("output_config") + if isinstance(existing_output_config, dict): + existing_output_config["format"] = schema_format + else: + request_body["output_config"] = {"format": schema_format} # rebind-ok: out-param # mutable-ok: json + return + + verbose_logger.warning( + "Bedrock Invoke: model=%s does not advertise `supports_native_structured_output` " + "in model_prices_and_context_window.json, so the JSON schema was inlined into " + "the last user message and is NOT enforced by the model.", + model, + ) + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=schema_format, + request_body=request_body, + ) + + +def strip_unsupported_bedrock_invoke_output_config_keys( + model: str, + request_body: dict[str, object], # mutable-ok: edited in place like siblings +) -> None: + """ + Drop ``output_config`` keys the Bedrock model does not accept. + + ``format`` survives unconditionally: it is only attached for models whose map + entry advertises ``supports_native_structured_output``. Effort-bearing keys + survive only when the map flags ``supports_output_config`` or a + ``supports_*_reasoning_effort`` tier; otherwise they are dropped with a + warning so Bedrock does not reject the request. + """ + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + output_config: Final = request_body.get("output_config") + if not isinstance(output_config, dict): + return + if all(key == "format" for key in output_config): + return + if _bedrock_model_supports(model, "supports_output_config") or AnthropicConfig._model_supports_effort_param( + model, "bedrock" + ): + return + + verbose_logger.warning( + "Bedrock Invoke: stripping unsupported `output_config` keys for " + "model=%s: neither `supports_output_config` nor any " + "`supports_*_reasoning_effort` flag is set in " + "model_prices_and_context_window.json. Add the capability " + "flag to the model JSON entry if this model accepts " + "`output_config`.", + model, + ) + preserved_format: Final = output_config.get("format") + if preserved_format is None: + request_body.pop("output_config", None) + else: + request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param # mutable-ok: json + + def normalize_custom_field_on_tools(request_body: dict) -> None: """ Drop the ``custom`` field from each tool, first hoisting a boolean diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f74a290d773..6ff9f0155f9 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -29,14 +29,14 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( - convert_bedrock_invoke_output_format_to_inline_schema, + apply_bedrock_invoke_structured_output, ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, is_claude_4_5_on_bedrock, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, - pop_bedrock_invoke_output_config_format, + strip_unsupported_bedrock_invoke_output_config_keys, ) from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, @@ -51,7 +51,6 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import GenericStreamingChunk, ModelResponseStream from litellm.types.utils import GenericStreamingChunk as GChunk -from litellm.utils import _supports_factory if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -708,52 +707,25 @@ class AmazonAnthropicClaudeMessagesConfig( # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) self._remove_ttl_from_cache_control(anthropic_messages_request=anthropic_messages_request, model=model) - # 5. Convert structured-output params to inline schema. - # Bedrock Invoke doesn't support top-level `output_format`; its - # accepted `output_config` subset is also narrower than Anthropic's, so - # consume the newer `output_config.format` shape here instead of - # forwarding it as an unknown nested key. + # 5. Route structured-output params (`output_format` / + # `output_config.format`) to native enforcement or the inline-schema + # fallback, then strip `output_config` keys the model does not accept. + # Ref: https://github.com/BerriAI/litellm/issues/22797 existing_output_config: Final = anthropic_messages_request.get("output_config") if isinstance(existing_output_config, dict): anthropic_messages_request["output_config"] = dict(existing_output_config) - output_format: Final = anthropic_messages_request.pop("output_format", None) - output_config_format: Final = pop_bedrock_invoke_output_config_format(anthropic_messages_request) - if output_format: - convert_bedrock_invoke_output_format_to_inline_schema( - output_format=output_format, - request_body=anthropic_messages_request, - ) - elif output_config_format: - convert_bedrock_invoke_output_format_to_inline_schema( - output_format=output_config_format, - request_body=anthropic_messages_request, - ) + apply_bedrock_invoke_structured_output( + model=model, + request_body=anthropic_messages_request, + ) normalize_bedrock_opus_output_config_effort( model=model, output_config=anthropic_messages_request.get("output_config"), ) - - # 5a. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models, - # but older models do not — strip it to avoid request rejection. - # Ref: https://github.com/BerriAI/litellm/issues/22797 - if not ( - _supports_factory( - model=model, - custom_llm_provider="bedrock", - key="supports_output_config", - ) - or AnthropicConfig._model_supports_effort_param(model, "bedrock") - ): - if anthropic_messages_request.pop("output_config", None) is not None: - verbose_logger.warning( - "Bedrock Invoke: stripping unsupported `output_config` for " - "model=%s — neither `supports_output_config` nor any " - "`supports_*_reasoning_effort` flag is set in " - "model_prices_and_context_window.json. Add the capability " - "flag to the model JSON entry if this model accepts " - "`output_config`.", - model, - ) + strip_unsupported_bedrock_invoke_output_config_keys( + model=model, + request_body=anthropic_messages_request, + ) # 5b. Hoist `custom.defer_loading` then drop `custom` (Bedrock doesn't support it) # Ref: https://github.com/BerriAI/litellm/issues/22847 @@ -774,9 +746,11 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + remaining_output_config: Final = anthropic_messages_request.get("output_config") if ( litellm.drop_params is True - and "output_config" in anthropic_messages_request + and isinstance(remaining_output_config, dict) + and any(key != "format" for key in remaining_output_config) and not AnthropicConfig._model_supports_effort_param(model, "bedrock") ): verbose_logger.warning( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f571292b015..27ff525c15e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1592,7 +1592,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1628,7 +1628,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1664,7 +1664,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1700,7 +1700,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1736,7 +1736,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1772,7 +1772,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2065,7 +2065,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2102,7 +2102,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2139,7 +2139,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2176,7 +2176,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2213,7 +2213,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2250,7 +2250,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f571292b015..27ff525c15e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1592,7 +1592,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1628,7 +1628,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1664,7 +1664,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1700,7 +1700,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1736,7 +1736,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -1772,7 +1772,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2065,7 +2065,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2102,7 +2102,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2139,7 +2139,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2176,7 +2176,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2213,7 +2213,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2250,7 +2250,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 25e2c3cda80..c4df46dea83 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -6207,3 +6207,45 @@ def test_disabled_thinking_omitted_only_for_always_on_models( assert "thinking" not in request else: assert request["thinking"] == {"type": "disabled"} + + +def test_anthropic_drop_params_keeps_format_only_output_config(monkeypatch): + """``drop_params=True`` must not consume ``output_config.format``: the drop + gate is an effort gate and ``format`` is a structured-output field.""" + monkeypatch.setattr(litellm, "drop_params", True) + config = AnthropicConfig() + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"z": {"type": "integer"}}}, + } + + result = config.transform_request( + model="claude-3-haiku-20240307", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"output_config": {"format": schema_format}}, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} + + +def test_anthropic_drop_params_reduces_mixed_output_config_to_format(monkeypatch): + """``drop_params=True`` drops the effort key on unsupported models but keeps + ``format`` so structured outputs still reach the provider.""" + monkeypatch.setattr(litellm, "drop_params", True) + config = AnthropicConfig() + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"z": {"type": "integer"}}}, + } + + result = config.transform_request( + model="claude-3-haiku-20240307", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"output_config": {"effort": "low", "format": schema_format}}, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index cea299280f8..a122d97a0f0 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -428,30 +428,58 @@ def test_output_config_forwarded_for_bedrock_chat_invoke_request(): def test_output_config_format_converted_for_bedrock_chat_invoke_request(): - """Bedrock Invoke chat path consumes ``output_config.format`` before forwarding.""" + """Bedrock Invoke chat path inlines ``output_config.format`` for models + without native structured-output support and keeps the effort key.""" config = AmazonAnthropicClaudeConfig() schema = { "type": "object", "properties": {"answer": {"type": "string"}}, } - result = config.transform_request( + with patch( # test-quality-ok: pin non-native path + "litellm.llms.bedrock.common_utils._bedrock_model_supports", + side_effect=lambda _model, key: key == "supports_output_config", + ): + result = config.transform_request( + model="anthropic.claude-opus-4-7", + messages=[{"role": "user", "content": "test"}], + optional_params={ + "max_tokens": 100, + "output_config": { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + }, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"effort": "xhigh"} + last_content = result["messages"][0]["content"] + assert json.loads(last_content[-1]["text"]) == schema + + +def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(): + """Bedrock Invoke chat path forwards ``output_config.format`` alongside effort + for models with native structured-output support (Claude Opus 4.7).""" + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + } + + result = AmazonAnthropicClaudeConfig().transform_request( model="anthropic.claude-opus-4-7", messages=[{"role": "user", "content": "test"}], optional_params={ "max_tokens": 100, - "output_config": { - "effort": "xhigh", - "format": {"type": "json_schema", "schema": schema}, - }, + "output_config": {"effort": "xhigh", "format": schema_format}, }, litellm_params={}, headers={}, ) - assert result.get("output_config") == {"effort": "xhigh"} - last_content = result["messages"][0]["content"] - assert json.loads(last_content[-1]["text"]) == schema + assert result.get("output_config") == {"effort": "xhigh", "format": schema_format} + assert "answer" not in json.dumps(result["messages"]) @pytest.mark.parametrize( @@ -488,7 +516,7 @@ def test_bedrock_chat_invoke_checks_output_config_support_with_bedrock_provider( optional_params = {"max_tokens": 100, "output_config": {"effort": "high"}} with patch( - "litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ) as mock_supports_factory: result = config.transform_request( @@ -499,11 +527,7 @@ def test_bedrock_chat_invoke_checks_output_config_support_with_bedrock_provider( headers={}, ) - mock_supports_factory.assert_called_once_with( - model="us.anthropic.claude-opus-4-7", - custom_llm_provider="bedrock", - key="supports_output_config", - ) + mock_supports_factory.assert_called_once_with("us.anthropic.claude-opus-4-7", "supports_output_config") assert result["output_config"] == {"effort": "high"} @@ -542,3 +566,80 @@ def test_output_format_removed_from_bedrock_invoke_request(): assert ( "output_format" not in result ), f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}" + + +def test_bedrock_chat_invoke_forwards_output_config_format_natively(local_model_cost_map): + """Regression: ``output_config.format`` is forwarded verbatim on models Bedrock + enforces structured outputs for, instead of being inlined as prompt text.""" + import json + + config = AmazonAnthropicClaudeConfig() + schema_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"zebra_count": {"type": "integer"}}, + "required": ["zebra_count"], + "additionalProperties": False, + }, + } + + result = config.transform_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "say hello"}], + optional_params={ + "max_tokens": 100, + "output_config": {"format": schema_format}, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} + assert "zebra_count" not in json.dumps(result["messages"]) + + +def test_bedrock_chat_invoke_drop_params_keeps_native_output_config_format(local_model_cost_map, monkeypatch): + """``drop_params=True`` must not eat ``output_config.format`` before the + native-forwarding router runs (Sonnet 4.5 has no effort flags).""" + import litellm + + monkeypatch.setattr(litellm, "drop_params", True) + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + } + + result = AmazonAnthropicClaudeConfig().transform_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "say hello"}], + optional_params={"max_tokens": 100, "output_config": {"format": schema_format}}, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} + + +def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_model_cost_map, monkeypatch): + """``drop_params=True`` on a model without native structured-output support + still reaches the inline-schema fallback instead of losing the schema.""" + import litellm + + monkeypatch.setattr(litellm, "drop_params", True) + schema = {"type": "object", "properties": {"zebra_count": {"type": "integer"}}} + + result = AmazonAnthropicClaudeConfig().transform_request( + model="anthropic.claude-3-haiku-20240307-v1:0", + messages=[{"role": "user", "content": "say hello"}], + optional_params={ + "max_tokens": 100, + "output_config": {"format": {"type": "json_schema", "schema": schema}}, + }, + litellm_params={}, + headers={}, + ) + + assert "output_config" not in result + last_content = result["messages"][-1]["content"] + assert json.loads(last_content[-1]["text"]) == schema diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 8d07d38b1b6..09ebc1a3c95 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -935,7 +935,7 @@ def test_bedrock_messages_strips_output_config(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=False, ): result = cfg.transform_anthropic_messages_request( @@ -970,7 +970,7 @@ def test_bedrock_messages_preserves_output_config_for_claude_4_6(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1003,7 +1003,7 @@ def test_bedrock_messages_checks_output_config_support_with_bedrock_provider(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ) as mock_supports_factory: result = cfg.transform_anthropic_messages_request( @@ -1014,11 +1014,7 @@ def test_bedrock_messages_checks_output_config_support_with_bedrock_provider(): headers={}, ) - mock_supports_factory.assert_called_with( - model="us.anthropic.claude-opus-4-7", - custom_llm_provider="bedrock", - key="supports_output_config", - ) + mock_supports_factory.assert_called_with("us.anthropic.claude-opus-4-7", "supports_output_config") assert result["output_config"] == {"effort": "high"} @@ -1038,7 +1034,7 @@ def test_bedrock_messages_forwards_output_config(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1054,27 +1050,29 @@ def test_bedrock_messages_forwards_output_config(): def test_bedrock_messages_forwards_output_config_with_output_format(): - """``output_config`` is forwarded; ``output_format`` is converted to inline schema.""" + """Legacy ``output_format`` is forwarded as ``output_config.format`` on models + that support native structured outputs, alongside the effort key.""" from unittest.mock import patch from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + schema_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + }, + } optional_params = { "max_tokens": 4096, "output_config": {"effort": "low"}, - "output_format": { - "type": "json_schema", - "schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - }, + "output_format": schema_format, } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1085,12 +1083,14 @@ def test_bedrock_messages_forwards_output_config_with_output_format(): headers={}, ) - assert result.get("output_config") == {"effort": "low"} + assert result.get("output_config") == {"effort": "low", "format": schema_format} assert "output_format" not in result + assert "answer" not in json.dumps(result["messages"]) def test_bedrock_messages_converts_output_config_format_to_inline_schema(): - """``output_config.format`` is consumed so Bedrock does not see an unknown nested key.""" + """Without native structured-output support, ``output_config.format`` falls back + to the inline schema so Bedrock does not see an unknown nested key.""" from unittest.mock import patch from litellm.types.router import GenericLiteLLMParams @@ -1110,8 +1110,8 @@ def test_bedrock_messages_converts_output_config_format_to_inline_schema(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", - return_value=True, + "litellm.llms.bedrock.common_utils._bedrock_model_supports", + side_effect=lambda _model, key: key == "supports_output_config", ): result = cfg.transform_anthropic_messages_request( model="anthropic.claude-opus-4-7", @@ -1146,7 +1146,7 @@ def test_bedrock_messages_normalizes_output_config_effort_for_opus( cfg = AmazonAnthropicClaudeMessagesConfig() with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1184,8 +1184,8 @@ def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", - return_value=True, + "litellm.llms.bedrock.common_utils._bedrock_model_supports", + side_effect=lambda _model, key: key == "supports_output_config", ): result = cfg.transform_anthropic_messages_request( model="anthropic.claude-opus-4-7", @@ -1229,7 +1229,7 @@ def test_bedrock_messages_does_not_mutate_callers_output_config(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): cfg.transform_anthropic_messages_request( @@ -1271,7 +1271,7 @@ def test_bedrock_messages_strips_output_config_with_output_format(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=False, ): result = cfg.transform_anthropic_messages_request( @@ -1332,7 +1332,7 @@ def test_bedrock_messages_drop_params_keeps_output_config_for_4_7(): litellm.drop_params = True try: with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1375,7 +1375,7 @@ def test_bedrock_messages_maps_reasoning_effort_for_adaptive_model( } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -1482,7 +1482,7 @@ def test_bedrock_messages_explicit_output_config_wins_over_reasoning_effort(): } with patch( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + "litellm.llms.bedrock.common_utils._bedrock_model_supports", return_value=True, ): result = cfg.transform_anthropic_messages_request( @@ -3104,3 +3104,149 @@ async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect(): break await asyncio.sleep(0.01) assert logging_obj.completion_start_time is not None + + +def test_bedrock_messages_forwards_output_config_format_natively(local_model_cost_map): + """Regression: on a model Bedrock enforces structured outputs for (Claude + Sonnet 4.5), ``output_config.format`` must be forwarded verbatim, not + silently rewritten into inline prompt text.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + schema_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "zebra_count": {"type": "integer"}, + "is_tuesday": {"type": "boolean"}, + }, + "required": ["zebra_count", "is_tuesday"], + "additionalProperties": False, + }, + } + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 100, + "output_config": {"format": schema_format}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} + assert "zebra_count" not in json.dumps(result["messages"]) + + +def test_bedrock_messages_inlines_schema_for_claude_5(local_model_cost_map): + """Bedrock rejects ``output_config.format`` for the Claude 5 family, so the + schema falls back to the inline-text path instead of a deterministic 400.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + schema = { + "type": "object", + "properties": {"zebra_count": {"type": "integer"}}, + } + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-5", + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 100, + "output_config": {"format": {"type": "json_schema", "schema": schema}}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "output_config" not in result + last_content = result["messages"][-1]["content"] + assert json.loads(last_content[-1]["text"]) == schema + + +def test_bedrock_messages_legacy_output_format_wins_over_output_config_format(local_model_cost_map): + """When a request carries both schema forms, the legacy top-level + ``output_format`` keeps winning, matching the pre-existing precedence.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + legacy_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"legacy_field": {"type": "string"}}}, + } + newer_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"newer_field": {"type": "string"}}}, + } + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 100, + "output_format": legacy_format, + "output_config": {"format": newer_format}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"format": legacy_format} + assert "output_format" not in result + assert "newer_field" not in json.dumps(result) + + +def test_bedrock_messages_drop_params_keeps_native_output_config_format(local_model_cost_map, monkeypatch): + """``drop_params=True`` must not strip a natively forwarded + ``output_config.format`` on models without effort support (Sonnet 4.5).""" + import litellm + from litellm.types.router import GenericLiteLLMParams + + monkeypatch.setattr(litellm, "drop_params", True) + cfg = AmazonAnthropicClaudeMessagesConfig() + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + } + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 100, + "output_config": {"format": schema_format}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} + + +def test_bedrock_messages_strips_effort_but_keeps_format_for_sonnet_4_5(local_model_cost_map): + """Sonnet 4.5 has native structured-output support but no effort support, so + a mixed ``output_config`` keeps ``format`` and drops ``effort``.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + schema_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + } + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "output_config": {"format": schema_format, "effort": "high"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"format": schema_format} diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index afd5e83ca52..9302dc01abe 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -522,6 +522,47 @@ def test_merge_bedrock_aws_request_params_keeps_caller_credentials_without_stati assert merged["aws_region_name"] == "us-west-2" +def test_strip_unsupported_output_config_keeps_format_drops_effort(local_model_cost_map): + """On a model with neither effort flag, only the ``format`` key survives.""" + from litellm.llms.bedrock.common_utils import ( + strip_unsupported_bedrock_invoke_output_config_keys, + ) + + schema_format = {"type": "json_schema", "schema": {"type": "object"}} + body = {"output_config": {"effort": "high", "format": schema_format}} + + strip_unsupported_bedrock_invoke_output_config_keys( + model="anthropic.claude-3-haiku-20240307-v1:0", + request_body=body, + ) + + assert body["output_config"] == {"format": schema_format} + + +def test_apply_structured_output_prefers_legacy_output_format(local_model_cost_map): + """The legacy ``output_format`` wins over ``output_config.format`` when a + request carries both, matching the pre-existing precedence.""" + from litellm.llms.bedrock.common_utils import ( + apply_bedrock_invoke_structured_output, + ) + + legacy = {"type": "json_schema", "schema": {"type": "object", "properties": {"a": {"type": "string"}}}} + newer = {"type": "json_schema", "schema": {"type": "object", "properties": {"b": {"type": "string"}}}} + body = { + "messages": [{"role": "user", "content": "hi"}], + "output_format": legacy, + "output_config": {"format": newer}, + } + + apply_bedrock_invoke_structured_output( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + request_body=body, + ) + + assert body["output_config"] == {"format": legacy} + assert "output_format" not in body + + def test_sign_aws_request_assumes_role_with_external_id(monkeypatch): """A trust policy requiring sts:ExternalId must be satisfied when signing batch API requests.""" import datetime