fix(bedrock): send native structured output for json_schema on Claude models that cannot force a tool (#49703)

* fix(bedrock): send native structured output for json_schema on Claude models that cannot force a tool

Opus 5.5 and Sonnet 5.5 on Bedrock cannot force the json tool, so a json_schema request without thinking
sent an unforced json tool the model ignored. The opt-in native output key now also applies when the model
map says the model cannot force a tool, and the Opus 5.5 commercial Bedrock rows carry the opt-in key. The
GovCloud Opus 5.5 rows carry it as false, like the Sonnet 5.5 GovCloud rows

* fix(bedrock): keep json tool with thinking on Opus 5.5 and Fable 5.1, and GovCloud on the tool

Native output without thinking now applies to every Bedrock Claude model that cannot
force the json tool unless supports_bedrock_native_structured_output is false, so Opus 5.5
needs no opt-in row and keeps the json tool when thinking is requested, as #46870 chose.
The invoke chat path keeps the native output_format it built instead of inlining it, and
GovCloud region-path ids read their own false rows.

* fix(bedrock): clean caller output_format schemas before sending them natively on invoke

* fix(bedrock): keep Mantle Claude on the json tool and send a caller's output_config.format natively on invoke

* fix(bedrock): keep the faked Converse stream for native output that replaces the json tool

A json_schema stream that goes native only because the model can't force the json tool now keeps the json tool's faked stream, so it still needs only bedrock:InvokeModel. /v1/messages builds a native-output retry only where it sends native output, and the Mantle test drops its mutable capture.

* test(bedrock): cover native output and the opt-out in the invoke json-mode refusal tests

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tavren27 2026-11-02 17:56:52 -08:00 • committed by GitHub
parent 183212e8be
commit 94b5b6386a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 518 additions and 45 deletions

View file

@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
from litellm.llms.bedrock.common_utils import (
bedrock_chat_opts_into_native_structured_output,
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
@ -934,8 +935,9 @@ class AmazonConverseConfig(BaseConfig):
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
request asks for thinking, which the json tool path would drop.
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the
request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot
force the json tool gets it unless that key is false.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
return bedrock_chat_uses_native_structured_output(
@ -1316,19 +1318,15 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text":
return optional_params
thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params))
native_schema: Final = (
self._native_output_schema(
model,
json_schema,
self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
model_info,
)
self._native_output_schema(model, json_schema, thinking_requested, model_info)
if json_schema is not None
else None
)
if native_schema is not None:
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
# No synthetic tool injection, no fake_stream needed.
# No synthetic tool injection.
# Requires an explicit schema — json_object with no schema falls through
# to the tool-call path below.
output_config: Final = self._create_output_config_for_response_format(
@ -1354,8 +1352,6 @@ class AmazonConverseConfig(BaseConfig):
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
)
if non_default_params.get("stream", False) is True:
optional_params["fake_stream"] = True
# else: response_format=json_object with no schema.
# Don't inject the synthetic json_tool_call tool here. When no
# schema is given, _create_json_tool_call_for_response_format
@ -1363,6 +1359,13 @@ class AmazonConverseConfig(BaseConfig):
# returns {} instead of the requested JSON. The model already
# returns JSON when the prompt asks for it.
# A request that goes native only because the model can't force the json tool keeps the json tool's faked
# stream, so it still needs only bedrock:InvokeModel.
streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output(
model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info
)
if json_schema is not None and not streams_natively and non_default_params.get("stream", False) is True:
optional_params["fake_stream"] = True
optional_params["json_mode"] = True
return optional_params

View file

@ -129,10 +129,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
return bedrock_chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
return self.chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def chat_uses_native_structured_output(
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
) -> bool:
return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info)
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
@ -335,10 +340,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
anthropic_request.pop("model", None)
anthropic_request.pop("stream", None)
model_info: Final = deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params))
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_request,
model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
model_info=model_info,
chat_uses_native_output=lambda: self.chat_uses_native_structured_output(
model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info
),
)
strip_unsupported_bedrock_invoke_output_config_keys(
model=model,

View file

@ -15,7 +15,7 @@ from litellm.llms.anthropic.chat.transformation import JsonModeRequest
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
AmazonAnthropicClaudeConfig,
)
from litellm.llms.bedrock.common_utils import build_mantle_messages_url
from litellm.llms.bedrock.common_utils import bedrock_chat_opts_into_native_structured_output, build_mantle_messages_url
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
@ -54,6 +54,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
region=region,
)
def chat_uses_native_structured_output(
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
) -> bool:
return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info)
def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params))

View file

@ -10,7 +10,7 @@ import json
import os
import re
import warnings
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -315,7 +316,33 @@ def bedrock_supports_native_structured_output(
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def bedrock_chat_uses_native_structured_output(
def _bedrock_map_native_output_flag(model: str) -> object:
routed: Final = strip_bedrock_routing_prefix(model)
flags: Final = (
(entry or {}).get("supports_bedrock_native_structured_output")
for entry in (
_model_cost_entry(f"bedrock/{routed}"),
_model_cost_entry(routed),
_model_cost_entry(split_bedrock_region_path(model)[1]),
)
)
return next((flag for flag in flags if isinstance(flag, bool)), None)
def _bedrock_native_output_opted_out(model: str, model_info: Mapping[str, object] | None) -> bool:
deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
if isinstance(deployment_flag, bool):
return not deployment_flag
return _bedrock_map_native_output_flag(model) is False
def _bedrock_native_output_replaces_unforceable_tool(model: str, model_info: Mapping[str, object] | None) -> bool:
return AnthropicModelInfo.forced_tool_use_unsupported(model) and not _bedrock_native_output_opted_out(
model, model_info
)
def bedrock_chat_opts_into_native_structured_output(
model: str,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
@ -326,7 +353,18 @@ def bedrock_chat_uses_native_structured_output(
)
def bedrock_native_output_is_opt_in(
def bedrock_chat_uses_native_structured_output(
model: str,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return bedrock_chat_opts_into_native_structured_output(
model, thinking_requested, custom_llm_provider, model_info
) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info))
def bedrock_native_output_is_deployment_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(
@ -334,6 +372,15 @@ def bedrock_native_output_is_opt_in(
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def bedrock_native_output_is_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
_bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
)
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return {
**litellm_params,
@ -424,13 +471,18 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
return False
def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
anthropic_optional_params
)
def requests_opt_in_native_output(
model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
) -> bool:
native_output_requested: Final = isinstance(
anthropic_optional_params.get("output_format"), Mapping
) or requests_output_config_format(anthropic_optional_params)
return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
return requests_native_output(anthropic_optional_params) and bedrock_native_output_is_opt_in(
model, model_info=model_info
)
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
@ -491,6 +543,7 @@ def apply_bedrock_invoke_structured_output(
model: str,
request_body: dict[str, object], # mutable-ok: edited in place like siblings
model_info: Mapping[str, object] | None = None,
chat_uses_native_output: Callable[[], bool] = lambda: False,
) -> None:
"""
Route Anthropic structured-output params to what the Bedrock model supports.
@ -502,7 +555,8 @@ def apply_bedrock_invoke_structured_output(
forwarded as ``output_config.format``, which Bedrock relays to the model for
enforced structured output. For every other model the schema is inlined into
the last user message as best-effort text, with a warning because nothing
enforces it.
enforces it. ``chat_uses_native_output`` is asked only when a schema is
present, and a true answer sends it natively after the same cleanup.
"""
legacy_output_format: Final = request_body.pop("output_format", None)
output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
@ -511,7 +565,15 @@ def apply_bedrock_invoke_structured_output(
if schema_format is None:
return
native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
native_format: Final = bedrock_native_output_format(
model,
schema_format,
model_info=(
{**(model_info or {}), "supports_bedrock_native_structured_output": True}
if chat_uses_native_output()
else model_info
),
)
if native_format is not None:
existing_output_config: Final = request_body.get("output_config")
if isinstance(existing_output_config, dict):

View file

@ -40,6 +40,7 @@ from litellm.llms.bedrock.common_utils import (
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
BedrockError,
apply_bedrock_invoke_structured_output,
bedrock_native_output_is_deployment_opt_in,
bedrock_supports_tool_search,
ensure_bedrock_anthropic_messages_tool_names,
get_anthropic_beta_from_headers,
@ -47,7 +48,7 @@ from litellm.llms.bedrock.common_utils import (
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_opt_in_native_output,
requests_native_output,
requests_output_config_format,
strip_unsupported_bedrock_invoke_output_config_keys,
supports_bedrock_prompt_cache_ttl,
@ -131,7 +132,10 @@ class AmazonAnthropicClaudeMessagesConfig(
litellm_params: GenericLiteLLMParams,
) -> GenericLiteLLMParams | None:
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
if not (
requests_native_output(anthropic_messages_optional_request_params)
and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
):
return None
return litellm_params.model_copy(
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}

View file

@ -49635,6 +49635,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -49679,6 +49680,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -68344,6 +68346,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -68389,6 +68392,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -68617,6 +68621,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -68657,6 +68662,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",

View file

@ -49635,6 +49635,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -49679,6 +49680,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -68344,6 +68346,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -68389,6 +68392,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -68617,6 +68621,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -68657,6 +68662,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",

View file

@ -201,20 +201,27 @@ def _marker() -> str:
return uuid.uuid4().hex
def _bedrock_invoke_deployment(
scenario: Scenario, wire_url: str, model_info: Mapping[str, JsonValue] | None = None
) -> str:
return scenario.model(
model=f"bedrock/invoke/{BEDROCK_MODEL}",
model_info=model_info,
api_base=wire_url,
aws_access_key_id="AKIASCRIPTEDPROVIDER",
aws_secret_access_key="scripted-secret",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire_url,
)
def _deployment(scenario: Scenario, provider: Provider, rig: Rig) -> str:
wire_url: Final = rig.wire.url
match provider:
case "anthropic":
return scenario.model(model=f"anthropic/{MODEL}", api_base=wire_url, api_key=_ANTHROPIC_KEY)
case "bedrock_invoke":
return scenario.model(
model=f"bedrock/invoke/{BEDROCK_MODEL}",
api_base=wire_url,
aws_access_key_id="AKIASCRIPTEDPROVIDER",
aws_secret_access_key="scripted-secret",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire_url,
)
return _bedrock_invoke_deployment(scenario, wire_url)
case "claude_platform":
return scenario.model(
model=f"bedrock/claude_platform/{MODEL}",
@ -1184,7 +1191,7 @@ def test_stream_cache_hit_skips_cached_provider_fields_that_are_not_an_object(ri
assert tuple(_message_fields(event["response"]) for event in stale_logs) == ((),) * len(stale_logs), stale_logs
JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke"]
JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke_native", "bedrock_invoke_tool"]
_UNMAPPED_MODEL: Final = "claude-scripted-9"
_JSON_ANSWER: Final = json.dumps({"answer": DOCS_PARTIAL_TEXT})
@ -1216,11 +1223,20 @@ _JSON_MODE_CASES: Final[Mapping[str, JsonModeCase]] = {
"anthropic_tool_enveloped": JsonModeCase(
"anthropic_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
),
"bedrock_invoke_native": JsonModeCase(
"bedrock_invoke_native",
"partial",
DOCS_PARTIAL_TEXT,
DOCS_PARTIAL_TEXT,
"content_filter",
"content_filter",
"native",
),
"bedrock_invoke_declared": JsonModeCase(
"bedrock_invoke", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
"bedrock_invoke_tool", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
),
"bedrock_invoke_enveloped": JsonModeCase(
"bedrock_invoke", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
"bedrock_invoke_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
),
}
@ -1234,8 +1250,15 @@ def _json_mode_deployment(scenario: Scenario, route: JsonModeRoute, rig: Rig) ->
scenario.model(model=f"anthropic/{_UNMAPPED_MODEL}", api_base=rig.wire.url, api_key=_ANTHROPIC_KEY),
"anthropic",
)
case "bedrock_invoke":
case "bedrock_invoke_native":
return _deployment(scenario, "bedrock_invoke", rig), "bedrock_invoke"
case "bedrock_invoke_tool":
return (
_bedrock_invoke_deployment(
scenario, rig.wire.url, {"supports_bedrock_native_structured_output": False}
),
"bedrock_invoke",
)
def _requested_format(body: Mapping[str, JsonValue]) -> tuple[bool, bool]:

View file

@ -20,6 +20,7 @@ _FABLE: Final = "us.anthropic.claude-fable-5"
_SONNET: Final = "us.anthropic.claude-sonnet-4-6"
_OPUS: Final = "us.anthropic.claude-opus-5-5"
_OPT_IN: Final = _OPUS
_UNFORCEABLE_TOOL: Final = _OPUS
_OPUS_4_8: Final = "us.anthropic.claude-opus-4-8"
_DEEPSEEK: Final = "deepseek.v3.2"
_AWS: Final[dict[str, JsonValue]] = {
@ -536,6 +537,12 @@ def _resolves(schema: Mapping[str, JsonValue], ref: str) -> bool:
return ref.startswith(prefix) and ref.removeprefix(prefix) in object_value(schema.get("$defs") or {})
def _tool_choice(body: Mapping[str, JsonValue], invoke: bool) -> JsonValue:
if invoke:
return body.get("tool_choice")
return object_value(body.get("toolConfig") or {}).get("toolChoice")
def _assert_native_without_tools(body: Mapping[str, JsonValue], invoke: bool, schema: Mapping[str, JsonValue]) -> None:
assert _native_schema(body, invoke) == schema, body
assert _JSON_TOOL not in _tools(body, invoke), body
@ -631,6 +638,40 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa
assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body
@pytest.mark.parametrize("route", ["converse", "invoke"])
def test_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool_gets_native_output(
gateway: Gateway, route: str
) -> None:
marker, marker_hex = _marker()
with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA))
)
assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text
_, body = _one_call(wire, marker_hex)
_assert_native_without_tools(body, route == "invoke", _SCHEMA)
@pytest.mark.parametrize("route", ["converse", "invoke"])
def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_output_with_an_unforced_json_tool(
gateway: Gateway, route: str
) -> None:
marker, marker_hex = _marker()
invoke: Final = route == "invoke"
with wire_server(_rejecting_native_output(_FABLE_REJECTION)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA))
)
assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text
rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain())
assert _native_schema(rejected, invoke) == _SCHEMA, rejected
assert _native_schema(retried, invoke) is None, retried
assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried
assert _tool_choice(retried, invoke) is None, retried
@pytest.mark.parametrize(
("backend", "model_info", "native"),
[
@ -797,6 +838,25 @@ def test_thinking_with_a_caller_tool_on_a_bedrock_native_model_sends_only_the_ca
assert _thinking_requested(body, False), body
def test_a_caller_tool_next_to_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool(
gateway: Gateway,
) -> None:
marker, marker_hex = _marker()
with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, "converse", _UNFORCEABLE_TOOL)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
_chat_body(model, marker, tools=[_USER_TOOL_SPEC], response_format=_response_format(_SCHEMA)),
)
(call,) = list_value(_chat_message(response)["tool_calls"])
function: Final = object_value(object_value(call)["function"])
assert function["name"] == _USER_TOOL, response.text
_, body = _one_call(wire, marker_hex)
assert _tools(body, False).keys() == {_USER_TOOL}, body
assert _native_schema(body, False) == _SCHEMA, body
@pytest.mark.parametrize(
("schema", "expected"),
[(_UNCLEANED_SCHEMA, _UNCLEANED_SCHEMA), (_NULLABLE_SCHEMA, _CLOSED_NULLABLE_SCHEMA)],

View file

@ -3,7 +3,7 @@ import copy
import json
import uuid
from types import SimpleNamespace
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from typing import Final
from unittest.mock import MagicMock, patch
@ -20,6 +20,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transform
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from tests.unit.conftest import NativeStructuredOutputKey
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
ONE_PIXEL_PNG = base64.b64decode(
@ -636,7 +637,6 @@ def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_nat
assert "tools" not in result
class Relabel(BaseModel):
model_config = ConfigDict(frozen=True)
@ -673,6 +673,7 @@ def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_re
assert item_ref.removeprefix("#/$defs/") in sent["$defs"]
assert "tools" not in result
def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map):
"""Bedrock Invoke chat path forwards ``output_config.format`` alongside effort
for models with native structured-output support (Claude Sonnet 4.6)."""
@ -1385,6 +1386,7 @@ def _vcr_outcome_gate(request, vcr):
yield
record_vcr_outcome(request, vcr)
@pytest.fixture(scope="session")
def event_loop():
try:
@ -1394,6 +1396,7 @@ def event_loop():
yield loop
loop.close()
@pytest.fixture(scope="function")
def setup_and_teardown(event_loop):
import litellm
@ -1427,6 +1430,7 @@ def setup_and_teardown(event_loop):
if pending:
event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
_SCALAR_DEFAULTS = {
"num_retries": getattr(litellm, "num_retries", None),
"set_verbose": getattr(litellm, "set_verbose", False),
@ -1498,6 +1502,7 @@ IN WITNESS WHEREOF, the parties have executed this Agreement.
* 8
) # Repeat to ensure we have enough tokens (need 1024+ for Claude models)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestBedrockAnthropicPromptCachingRegression:
"""
@ -1660,6 +1665,7 @@ class TestBedrockAnthropicPromptCachingRegression:
if "anthropic_beta" in additional_fields:
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestBedrockAnthropic1MContextRegression:
"""
@ -1845,6 +1851,7 @@ class TestBedrockAnthropic1MContextRegression:
assert "context-1m-2025-08-07" in beta_headers
assert "computer-use-2024-10-22" in beta_headers
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestBedrockAnthropicCombinedRegressions:
"""
@ -2011,6 +2018,104 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
)
@pytest.mark.parametrize(
("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"),
[
(False, None, None, {}, True),
(False, None, None, {"thinking": {"type": "adaptive"}}, False),
(False, True, None, {"thinking": {"type": "adaptive"}}, True),
(False, False, None, {}, False),
(False, None, {"supports_bedrock_native_structured_output": False}, {}, False),
(True, True, None, {}, False),
],
ids=[
"cannot-force-tool",
"cannot-force-tool-thinking",
"cannot-force-tool-thinking-key-true",
"cannot-force-tool-key-false",
"cannot-force-tool-deployment-opt-out",
"can-force-tool",
],
)
def test_invoke_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out(
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
monkeypatch: pytest.MonkeyPatch,
forced_tool_use: bool,
map_flag: bool | None,
model_info: Mapping[str, object] | None,
thinking_params: Mapping[str, object],
native: bool,
) -> None:
model: Final = "us.anthropic.claude-haiku-5-5"
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
set_native_structured_output(model, "bedrock" if map_flag else None)
if map_flag is None:
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
config: Final = AmazonAnthropicClaudeConfig(model_info=model_info)
optional_params: Final = config.map_openai_params(
non_default_params={"response_format": _CITY_RESPONSE_FORMAT, "max_tokens": 2048, **thinking_params},
optional_params={},
model=model,
drop_params=False,
)
body: Final = config.transform_request(
model=model,
messages=[{"role": "user", "content": "Capital of France?"}],
optional_params=optional_params,
litellm_params={} if model_info is None else {"model_info": model_info},
headers={},
)
assert (body.get("output_config", {}).get("format", {}).get("schema") == _CITY_SCHEMA) is native
assert ("tools" in body) is not native
@pytest.mark.parametrize(
"caller_schema_param",
[
lambda schema: {"output_format": {"type": "json_schema", "schema": schema}},
lambda schema: {"output_config": {"format": {"type": "json_schema", "schema": schema}}},
],
ids=["output_format", "output_config.format"],
)
@pytest.mark.parametrize(
("model", "forced_tool_use", "native_key"),
[("us.anthropic.claude-opus-4-8", True, "legacy"), ("us.anthropic.claude-haiku-5-5", False, None)],
ids=["native-key", "cannot-force-tool"],
)
def test_invoke_caller_schema_drops_unsupported_constraints_before_going_native(
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
monkeypatch: pytest.MonkeyPatch,
model: str,
forced_tool_use: bool,
native_key: NativeStructuredOutputKey,
caller_schema_param: Callable[[Mapping[str, object]], dict[str, object]],
) -> None:
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
set_native_structured_output(model, native_key)
if native_key is None:
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
schema: Final = {
"type": "object",
"properties": {"age": {"type": "integer", "minimum": 0}},
"required": ["age"],
"additionalProperties": False,
}
body: Final = AmazonAnthropicClaudeConfig().transform_request(
model=model,
messages=[{"role": "user", "content": "How old is the oldest tree?"}],
optional_params={**caller_schema_param(schema), "max_tokens": 2048},
litellm_params={},
headers={},
)
sent_age: Final = body["output_config"]["format"]["schema"]["properties"]["age"]
assert sent_age["type"] == "integer"
assert "minimum" not in sent_age
BEDROCK_HAIKU_45_BASE_ID: Final = "anthropic.claude-haiku-4-5-20251001-v1:0"
@ -2033,7 +2138,9 @@ def test_bedrock_invoke_claude_default_max_tokens_comes_from_base_model(local_mo
headers={},
)
assert body["max_tokens"] == mapped_cap, f"default max_tokens should be the base model's cap, got {body['max_tokens']}"
assert body["max_tokens"] == mapped_cap, (
f"default max_tokens should be the base model's cap, got {body['max_tokens']}"
)
def test_bedrock_invoke_claude_explicit_max_tokens_is_kept(local_model_cost_map):
@ -2093,7 +2200,9 @@ def test_bedrock_invoke_claude_default_max_tokens_yields_to_anthropic_config(loc
headers={},
)
assert body["max_tokens"] == configured_max_tokens, f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}"
assert body["max_tokens"] == configured_max_tokens, (
f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}"
)
@pytest.mark.parametrize(("configured_max_tokens", "deployment_cap"), [(100000, 50), (1000, 8192)])

View file

@ -3,6 +3,7 @@ import json
import uuid
from types import SimpleNamespace
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
@ -246,6 +247,39 @@ async def test_mantle_claude_async_completion_streams_a_wrapped_json_schema_answ
assert _streamed_answer(chunks) == BARE_STREAMED_ANSWER
def _json_tool_message(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "anthropic.claude-opus-5-5",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": RISK_ANSWER}],
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 9},
},
)
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("model", ("bedrock_mantle/anthropic.claude-opus-5-5", "bedrock/mantle/anthropic.claude-opus-5-5"))
def test_mantle_claude_that_cannot_force_a_tool_keeps_json_schema_on_the_json_tool(model: str) -> None:
respond: Final = MagicMock(wraps=_json_tool_message)
response: Final = litellm.completion(
**{**_json_schema_stream_kwargs(model), "stream": False},
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
)
sent: Final = tuple(json.loads(call.args[0].content) for call in respond.call_args_list)
assert json.loads(response.choices[0].message.content) == RISK_ANSWER
assert [(body.get("output_config", {}).get("format"), [tool["name"] for tool in body["tools"]]) for body in sent] == [
(None, ["json_tool_call"])
]
def test_mantle_claude_default_max_tokens_comes_from_the_mantle_row(local_model_cost_map):
from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig

View file

@ -4,7 +4,7 @@ import json
import logging
import os
import re
from collections.abc import Iterator, Mapping
from collections.abc import Callable, Iterator, Mapping
from typing import Final
from unittest.mock import MagicMock, Mock, patch
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_tools_pt,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.anthropic.chat.transformation import (
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES,
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS,
@ -31,6 +32,7 @@ from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTI
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject
from litellm.types.llms.openai import AllMessageValues
from tests.unit.conftest import NativeStructuredOutputKey
def test_transform_usage():
@ -4981,7 +4983,10 @@ ADAPTIVE_THINKING: Final = {"type": "adaptive"}
def _converse_request_for_json_schema(
model: str, stream: bool, thinking_params: Mapping[str, object] | None = None
model: str,
stream: bool,
thinking_params: Mapping[str, object] | None = None,
model_info: Mapping[str, object] | None = None,
) -> tuple[Mapping[str, object], RequestObject]:
config = AmazonConverseConfig()
optional_params = config.map_openai_params(
@ -4989,12 +4994,13 @@ def _converse_request_for_json_schema(
optional_params={},
model=model,
drop_params=False,
model_info=model_info,
)
request = config._transform_request(
model=model,
messages=[{"role": "user", "content": "Name the capital of France."}],
optional_params=optional_params,
litellm_params={},
litellm_params={} if model_info is None else {"model_info": model_info},
headers={},
)
return optional_params, request
@ -5061,6 +5067,115 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested(
assert ("toolConfig" in request) is not native
_FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}}
@pytest.mark.parametrize(
("forced_tool_use", "map_flag", "model_info", "thinking_params", "native", "tool_choice"),
[
(False, None, None, None, True, None),
(False, None, None, {"thinking": {"type": "adaptive"}}, False, None),
(False, True, None, {"thinking": {"type": "adaptive"}}, True, None),
(False, False, None, None, False, None),
(False, None, {"supports_bedrock_native_structured_output": False}, None, False, None),
(True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE),
],
ids=[
"cannot-force-tool",
"cannot-force-tool-thinking",
"cannot-force-tool-thinking-key-true",
"cannot-force-tool-key-false",
"cannot-force-tool-deployment-opt-out",
"can-force-tool",
],
)
def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out(
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
monkeypatch: pytest.MonkeyPatch,
forced_tool_use: bool,
map_flag: bool | None,
model_info: Mapping[str, object] | None,
thinking_params: Mapping[str, object] | None,
native: bool,
tool_choice: Mapping[str, object] | None,
) -> None:
model: Final = "us.anthropic.claude-haiku-5-5"
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
set_native_structured_output(model, "bedrock" if map_flag else None)
if map_flag is None:
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
_, request = _converse_request_for_json_schema(model, False, thinking_params, model_info=model_info)
assert ("outputConfig" in request) is native
assert ("toolConfig" in request) is not native
assert request.get("toolConfig", {}).get("toolChoice") == tool_choice
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
("model", "thinking_params", "native"),
[
("us.anthropic.claude-opus-5-5", None, True),
("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False),
("global.anthropic.claude-fable-5-1", None, True),
("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False),
("us-gov.anthropic.claude-opus-5-5", None, False),
("us-gov.anthropic.claude-fable-5-1", None, False),
("us-gov-west-1/anthropic.claude-opus-5-5", None, False),
("converse/us-gov-east-1/anthropic.claude-fable-5-1", None, False),
("us-east-1/anthropic.claude-opus-5-5", None, True),
],
)
def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking(
model: str, thinking_params: Mapping[str, object] | None, native: bool
) -> None:
_, request = _converse_request_for_json_schema(model, False, thinking_params)
assert ("outputConfig" in request) is native
assert ("toolConfig" in request) is not native
def _converse_capital_answer(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"output": {"message": {"role": "assistant", "content": [{"text": '{"city":"Paris","country":"France"}'}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 1, "outputTokens": 12, "totalTokens": 13},
},
request=request,
)
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
"model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"]
)
def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming(
model: str,
) -> None:
respond: Final = MagicMock(wraps=_converse_capital_answer)
stream: Final = litellm.completion(
model=f"bedrock/{model}",
messages=[{"role": "user", "content": "Name the capital of France."}],
response_format=CAPITAL_RESPONSE_FORMAT,
stream=True,
aws_access_key_id="AKIAEXAMPLE",
aws_secret_access_key="fake-secret",
aws_region_name="us-east-1",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
)
assert isinstance(stream, CustomStreamWrapper)
streamed: Final = "".join(chunk.choices[0].delta.content or "" for chunk in stream)
sent: Final = respond.call_args.args[0]
assert sent.url.path.endswith("/converse")
assert "textFormat" in json.loads(sent.content)["outputConfig"]
assert json.loads(streamed) == {"city": "Paris", "country": "France"}
def test_adaptive_thinking_is_kept_next_to_json_schema_on_a_native_output_profile(set_native_structured_output):
model: Final = "us.anthropic.claude-opus-5"
set_native_structured_output(model, "bedrock")

View file

@ -4175,6 +4175,43 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati
assert len(bedrock.bodies) == 1
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
("model", "model_info"),
[
("us.anthropic.claude-opus-5-5", None),
("global.anthropic.claude-fable-5-1", None),
("us.anthropic.claude-opus-4-8", None),
("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}),
],
ids=["opus_5_5_inlined", "fable_5_1_inlined", "mapped_opt_in", "deployment_opt_in"],
)
def test_bedrock_messages_offers_a_native_output_retry_only_when_it_sends_native_output(
model: str, model_info: Mapping[str, object] | None
) -> None:
from litellm.types.router import GenericLiteLLMParams
config: Final = AmazonAnthropicClaudeMessagesConfig()
litellm_params: Final = GenericLiteLLMParams(model_info=model_info)
optional_params: Final = {
"max_tokens": 100,
"output_config": {"format": {"type": "json_schema", "schema": _CONSTRAINED_SCHEMA}},
}
body: Final = REASONING_BODY_ADAPTER.validate_python(
config.transform_anthropic_messages_request(
model=model,
messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}],
anthropic_messages_optional_request_params=optional_params,
litellm_params=litellm_params,
headers={},
)
)
retry_params: Final = config.native_output_fallback_litellm_params(model, optional_params, litellm_params)
assert (retry_params is not None) is ("format" in REASONING_BODY_ADAPTER.validate_python(body.get("output_config", {})))
# Bedrock's 400 bodies for requests past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-29
_STRICT_TOOLS_REJECTIONS: Final = (
{