diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 9a345ccdc9d..2d185068815 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images from litellm.llms.bedrock.common_utils import ( + bedrock_chat_opts_into_native_structured_output, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, bedrock_model_is_unknown, @@ -934,8 +935,9 @@ class AmazonConverseConfig(BaseConfig): """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and - ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the - request asks for thinking, which the json tool path would drop. + ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the + request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot + force the json tool gets it unless that key is false. Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html """ return bedrock_chat_uses_native_structured_output( @@ -1316,19 +1318,15 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params + thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)) native_schema: Final = ( - self._native_output_schema( - model, - json_schema, - self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)), - model_info, - ) + self._native_output_schema(model, json_schema, thinking_requested, model_info) if json_schema is not None else None ) if native_schema is not None: # Use Bedrock's native structured outputs API (outputConfig.textFormat) - # No synthetic tool injection, no fake_stream needed. + # No synthetic tool injection. # Requires an explicit schema — json_object with no schema falls through # to the tool-call path below. output_config: Final = self._create_output_config_for_response_format( @@ -1354,8 +1352,6 @@ class AmazonConverseConfig(BaseConfig): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) ) - if non_default_params.get("stream", False) is True: - optional_params["fake_stream"] = True # else: response_format=json_object with no schema. # Don't inject the synthetic json_tool_call tool here. When no # schema is given, _create_json_tool_call_for_response_format @@ -1363,6 +1359,13 @@ class AmazonConverseConfig(BaseConfig): # returns {} instead of the requested JSON. The model already # returns JSON when the prompt asks for it. + # A request that goes native only because the model can't force the json tool keeps the json tool's faked + # stream, so it still needs only bedrock:InvokeModel. + streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output( + model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info + ) + if json_schema is not None and not streams_natively and non_default_params.get("stream", False) is True: + optional_params["fake_stream"] = True optional_params["json_mode"] = True return optional_params diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index e150791b09e..e2f12b123de 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -129,10 +129,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - return bedrock_chat_uses_native_structured_output( - model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info + return self.chat_uses_native_structured_output( + model, self.is_thinking_requested(request_params), self._deployment_model_info ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + def chat_uses_native_structured_output( + self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + ) -> bool: + return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info) + def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] ) -> bool: @@ -335,10 +340,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) + model_info: Final = deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)) apply_bedrock_invoke_structured_output( model=model, request_body=anthropic_request, - model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)), + model_info=model_info, + chat_uses_native_output=lambda: self.chat_uses_native_structured_output( + model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info + ), ) strip_unsupported_bedrock_invoke_output_config_keys( model=model, diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index cbde6f918bc..952ca7a2cd4 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -15,7 +15,7 @@ from litellm.llms.anthropic.chat.transformation import JsonModeRequest from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) -from litellm.llms.bedrock.common_utils import build_mantle_messages_url +from litellm.llms.bedrock.common_utils import bedrock_chat_opts_into_native_structured_output, build_mantle_messages_url from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -54,6 +54,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): region=region, ) + def chat_uses_native_structured_output( + self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + ) -> bool: + return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info) + def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]: return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params)) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index dca5e1c7a7b..3126450b287 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -10,7 +10,7 @@ import json import os import re import warnings -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -315,7 +316,33 @@ def bedrock_supports_native_structured_output( ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) -def bedrock_chat_uses_native_structured_output( +def _bedrock_map_native_output_flag(model: str) -> object: + routed: Final = strip_bedrock_routing_prefix(model) + flags: Final = ( + (entry or {}).get("supports_bedrock_native_structured_output") + for entry in ( + _model_cost_entry(f"bedrock/{routed}"), + _model_cost_entry(routed), + _model_cost_entry(split_bedrock_region_path(model)[1]), + ) + ) + return next((flag for flag in flags if isinstance(flag, bool)), None) + + +def _bedrock_native_output_opted_out(model: str, model_info: Mapping[str, object] | None) -> bool: + deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output") + if isinstance(deployment_flag, bool): + return not deployment_flag + return _bedrock_map_native_output_flag(model) is False + + +def _bedrock_native_output_replaces_unforceable_tool(model: str, model_info: Mapping[str, object] | None) -> bool: + return AnthropicModelInfo.forced_tool_use_unsupported(model) and not _bedrock_native_output_opted_out( + model, model_info + ) + + +def bedrock_chat_opts_into_native_structured_output( model: str, thinking_requested: bool, custom_llm_provider: str = "bedrock", @@ -326,7 +353,18 @@ def bedrock_chat_uses_native_structured_output( ) -def bedrock_native_output_is_opt_in( +def bedrock_chat_uses_native_structured_output( + model: str, + thinking_requested: bool, + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> bool: + return bedrock_chat_opts_into_native_structured_output( + model, thinking_requested, custom_llm_provider, model_info + ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info)) + + +def bedrock_native_output_is_deployment_opt_in( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: return not _bedrock_model_supports( @@ -334,6 +372,15 @@ def bedrock_native_output_is_opt_in( ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) +def bedrock_native_output_is_opt_in( + model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None +) -> bool: + return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and ( + _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + or _bedrock_native_output_replaces_unforceable_tool(model, model_info) + ) + + def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]: return { **litellm_params, @@ -424,13 +471,18 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool return False +def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool: + return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format( + anthropic_optional_params + ) + + def requests_opt_in_native_output( model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - native_output_requested: Final = isinstance( - anthropic_optional_params.get("output_format"), Mapping - ) or requests_output_config_format(anthropic_optional_params) - return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info) + return requests_native_output(anthropic_optional_params) and bedrock_native_output_is_opt_in( + model, model_info=model_info + ) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) @@ -491,6 +543,7 @@ def apply_bedrock_invoke_structured_output( model: str, request_body: dict[str, object], # mutable-ok: edited in place like siblings model_info: Mapping[str, object] | None = None, + chat_uses_native_output: Callable[[], bool] = lambda: False, ) -> None: """ Route Anthropic structured-output params to what the Bedrock model supports. @@ -502,7 +555,8 @@ def apply_bedrock_invoke_structured_output( forwarded as ``output_config.format``, which Bedrock relays to the model for enforced structured output. For every other model the schema is inlined into the last user message as best-effort text, with a warning because nothing - enforces it. + enforces it. ``chat_uses_native_output`` is asked only when a schema is + present, and a true answer sends it natively after the same cleanup. """ legacy_output_format: Final = request_body.pop("output_format", None) output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body) @@ -511,7 +565,15 @@ def apply_bedrock_invoke_structured_output( if schema_format is None: return - native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info) + native_format: Final = bedrock_native_output_format( + model, + schema_format, + model_info=( + {**(model_info or {}), "supports_bedrock_native_structured_output": True} + if chat_uses_native_output() + else model_info + ), + ) if native_format is not None: existing_output_config: Final = request_body.get("output_config") if isinstance(existing_output_config, dict): diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index f358dbe96db..8b4e2e5b5c5 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -40,6 +40,7 @@ from litellm.llms.bedrock.common_utils import ( BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS, BedrockError, apply_bedrock_invoke_structured_output, + bedrock_native_output_is_deployment_opt_in, bedrock_supports_tool_search, ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, @@ -47,7 +48,7 @@ from litellm.llms.bedrock.common_utils import ( normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, - requests_opt_in_native_output, + requests_native_output, requests_output_config_format, strip_unsupported_bedrock_invoke_output_config_keys, supports_bedrock_prompt_cache_ttl, @@ -131,7 +132,10 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params: GenericLiteLLMParams, ) -> GenericLiteLLMParams | None: model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info) - if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info): + if not ( + requests_native_output(anthropic_messages_optional_request_params) + and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info) + ): return None return litellm_params.model_copy( update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}} diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7a430148a53..cf8544316e9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49635,6 +49635,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -49679,6 +49680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -68344,6 +68346,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -68389,6 +68392,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -68617,6 +68621,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -68657,6 +68662,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7a430148a53..cf8544316e9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49635,6 +49635,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -49679,6 +49680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -68344,6 +68346,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -68389,6 +68392,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -68617,6 +68621,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -68657,6 +68662,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py b/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py index c9a85545d8b..8774d590b37 100644 --- a/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py +++ b/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py @@ -201,20 +201,27 @@ def _marker() -> str: return uuid.uuid4().hex +def _bedrock_invoke_deployment( + scenario: Scenario, wire_url: str, model_info: Mapping[str, JsonValue] | None = None +) -> str: + return scenario.model( + model=f"bedrock/invoke/{BEDROCK_MODEL}", + model_info=model_info, + api_base=wire_url, + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire_url, + ) + + def _deployment(scenario: Scenario, provider: Provider, rig: Rig) -> str: wire_url: Final = rig.wire.url match provider: case "anthropic": return scenario.model(model=f"anthropic/{MODEL}", api_base=wire_url, api_key=_ANTHROPIC_KEY) case "bedrock_invoke": - return scenario.model( - model=f"bedrock/invoke/{BEDROCK_MODEL}", - api_base=wire_url, - aws_access_key_id="AKIASCRIPTEDPROVIDER", - aws_secret_access_key="scripted-secret", - aws_region_name="us-east-1", - aws_bedrock_runtime_endpoint=wire_url, - ) + return _bedrock_invoke_deployment(scenario, wire_url) case "claude_platform": return scenario.model( model=f"bedrock/claude_platform/{MODEL}", @@ -1184,7 +1191,7 @@ def test_stream_cache_hit_skips_cached_provider_fields_that_are_not_an_object(ri assert tuple(_message_fields(event["response"]) for event in stale_logs) == ((),) * len(stale_logs), stale_logs -JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke"] +JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke_native", "bedrock_invoke_tool"] _UNMAPPED_MODEL: Final = "claude-scripted-9" _JSON_ANSWER: Final = json.dumps({"answer": DOCS_PARTIAL_TEXT}) @@ -1216,11 +1223,20 @@ _JSON_MODE_CASES: Final[Mapping[str, JsonModeCase]] = { "anthropic_tool_enveloped": JsonModeCase( "anthropic_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool" ), + "bedrock_invoke_native": JsonModeCase( + "bedrock_invoke_native", + "partial", + DOCS_PARTIAL_TEXT, + DOCS_PARTIAL_TEXT, + "content_filter", + "content_filter", + "native", + ), "bedrock_invoke_declared": JsonModeCase( - "bedrock_invoke", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool" + "bedrock_invoke_tool", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool" ), "bedrock_invoke_enveloped": JsonModeCase( - "bedrock_invoke", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool" + "bedrock_invoke_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool" ), } @@ -1234,8 +1250,15 @@ def _json_mode_deployment(scenario: Scenario, route: JsonModeRoute, rig: Rig) -> scenario.model(model=f"anthropic/{_UNMAPPED_MODEL}", api_base=rig.wire.url, api_key=_ANTHROPIC_KEY), "anthropic", ) - case "bedrock_invoke": + case "bedrock_invoke_native": return _deployment(scenario, "bedrock_invoke", rig), "bedrock_invoke" + case "bedrock_invoke_tool": + return ( + _bedrock_invoke_deployment( + scenario, rig.wire.url, {"supports_bedrock_native_structured_output": False} + ), + "bedrock_invoke", + ) def _requested_format(body: Mapping[str, JsonValue]) -> tuple[bool, bool]: diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 5803d1dee44..67edcf7f3e4 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -20,6 +20,7 @@ _FABLE: Final = "us.anthropic.claude-fable-5" _SONNET: Final = "us.anthropic.claude-sonnet-4-6" _OPUS: Final = "us.anthropic.claude-opus-5-5" _OPT_IN: Final = _OPUS +_UNFORCEABLE_TOOL: Final = _OPUS _OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _DEEPSEEK: Final = "deepseek.v3.2" _AWS: Final[dict[str, JsonValue]] = { @@ -536,6 +537,12 @@ def _resolves(schema: Mapping[str, JsonValue], ref: str) -> bool: return ref.startswith(prefix) and ref.removeprefix(prefix) in object_value(schema.get("$defs") or {}) +def _tool_choice(body: Mapping[str, JsonValue], invoke: bool) -> JsonValue: + if invoke: + return body.get("tool_choice") + return object_value(body.get("toolConfig") or {}).get("toolChoice") + + def _assert_native_without_tools(body: Mapping[str, JsonValue], invoke: bool, schema: Mapping[str, JsonValue]) -> None: assert _native_schema(body, invoke) == schema, body assert _JSON_TOOL not in _tools(body, invoke), body @@ -631,6 +638,40 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool_gets_native_output( + gateway: Gateway, route: str +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, route == "invoke", _SCHEMA) + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_output_with_an_unforced_json_tool( + gateway: Gateway, route: str +) -> None: + marker, marker_hex = _marker() + invoke: Final = route == "invoke" + with wire_server(_rejecting_native_output(_FABLE_REJECTION)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + assert _native_schema(rejected, invoke) == _SCHEMA, rejected + assert _native_schema(retried, invoke) is None, retried + assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried + assert _tool_choice(retried, invoke) is None, retried + + @pytest.mark.parametrize( ("backend", "model_info", "native"), [ @@ -797,6 +838,25 @@ def test_thinking_with_a_caller_tool_on_a_bedrock_native_model_sends_only_the_ca assert _thinking_requested(body, False), body +def test_a_caller_tool_next_to_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool( + gateway: Gateway, +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _UNFORCEABLE_TOOL) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, marker, tools=[_USER_TOOL_SPEC], response_format=_response_format(_SCHEMA)), + ) + (call,) = list_value(_chat_message(response)["tool_calls"]) + function: Final = object_value(object_value(call)["function"]) + assert function["name"] == _USER_TOOL, response.text + _, body = _one_call(wire, marker_hex) + assert _tools(body, False).keys() == {_USER_TOOL}, body + assert _native_schema(body, False) == _SCHEMA, body + + @pytest.mark.parametrize( ("schema", "expected"), [(_UNCLEANED_SCHEMA, _UNCLEANED_SCHEMA), (_NULLABLE_SCHEMA, _CLOSED_NULLABLE_SCHEMA)], diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 4850a402103..05d755b1215 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -3,7 +3,7 @@ import copy import json import uuid from types import SimpleNamespace -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Final from unittest.mock import MagicMock, patch @@ -20,6 +20,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transform ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +from tests.unit.conftest import NativeStructuredOutputKey from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE ONE_PIXEL_PNG = base64.b64decode( @@ -636,7 +637,6 @@ def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_nat assert "tools" not in result - class Relabel(BaseModel): model_config = ConfigDict(frozen=True) @@ -673,6 +673,7 @@ def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_re assert item_ref.removeprefix("#/$defs/") in sent["$defs"] assert "tools" not in result + def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map): """Bedrock Invoke chat path forwards ``output_config.format`` alongside effort for models with native structured-output support (Claude Sonnet 4.6).""" @@ -1385,6 +1386,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -1394,6 +1396,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -1427,6 +1430,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -1498,6 +1502,7 @@ IN WITNESS WHEREOF, the parties have executed this Agreement. * 8 ) # Repeat to ensure we have enough tokens (need 1024+ for Claude models) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockAnthropicPromptCachingRegression: """ @@ -1660,6 +1665,7 @@ class TestBedrockAnthropicPromptCachingRegression: if "anthropic_beta" in additional_fields: assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockAnthropic1MContextRegression: """ @@ -1845,6 +1851,7 @@ class TestBedrockAnthropic1MContextRegression: assert "context-1m-2025-08-07" in beta_headers assert "computer-use-2024-10-22" in beta_headers + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockAnthropicCombinedRegressions: """ @@ -2011,6 +2018,104 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) +@pytest.mark.parametrize( + ("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"), + [ + (False, None, None, {}, True), + (False, None, None, {"thinking": {"type": "adaptive"}}, False), + (False, True, None, {"thinking": {"type": "adaptive"}}, True), + (False, False, None, {}, False), + (False, None, {"supports_bedrock_native_structured_output": False}, {}, False), + (True, True, None, {}, False), + ], + ids=[ + "cannot-force-tool", + "cannot-force-tool-thinking", + "cannot-force-tool-thinking-key-true", + "cannot-force-tool-key-false", + "cannot-force-tool-deployment-opt-out", + "can-force-tool", + ], +) +def test_invoke_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out( + set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None], + monkeypatch: pytest.MonkeyPatch, + forced_tool_use: bool, + map_flag: bool | None, + model_info: Mapping[str, object] | None, + thinking_params: Mapping[str, object], + native: bool, +) -> None: + model: Final = "us.anthropic.claude-haiku-5-5" + monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use) + set_native_structured_output(model, "bedrock" if map_flag else None) + if map_flag is None: + monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output") + + config: Final = AmazonAnthropicClaudeConfig(model_info=model_info) + optional_params: Final = config.map_openai_params( + non_default_params={"response_format": _CITY_RESPONSE_FORMAT, "max_tokens": 2048, **thinking_params}, + optional_params={}, + model=model, + drop_params=False, + ) + body: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Capital of France?"}], + optional_params=optional_params, + litellm_params={} if model_info is None else {"model_info": model_info}, + headers={}, + ) + + assert (body.get("output_config", {}).get("format", {}).get("schema") == _CITY_SCHEMA) is native + assert ("tools" in body) is not native + + +@pytest.mark.parametrize( + "caller_schema_param", + [ + lambda schema: {"output_format": {"type": "json_schema", "schema": schema}}, + lambda schema: {"output_config": {"format": {"type": "json_schema", "schema": schema}}}, + ], + ids=["output_format", "output_config.format"], +) +@pytest.mark.parametrize( + ("model", "forced_tool_use", "native_key"), + [("us.anthropic.claude-opus-4-8", True, "legacy"), ("us.anthropic.claude-haiku-5-5", False, None)], + ids=["native-key", "cannot-force-tool"], +) +def test_invoke_caller_schema_drops_unsupported_constraints_before_going_native( + set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None], + monkeypatch: pytest.MonkeyPatch, + model: str, + forced_tool_use: bool, + native_key: NativeStructuredOutputKey, + caller_schema_param: Callable[[Mapping[str, object]], dict[str, object]], +) -> None: + monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use) + set_native_structured_output(model, native_key) + if native_key is None: + monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output") + schema: Final = { + "type": "object", + "properties": {"age": {"type": "integer", "minimum": 0}}, + "required": ["age"], + "additionalProperties": False, + } + + body: Final = AmazonAnthropicClaudeConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "How old is the oldest tree?"}], + optional_params={**caller_schema_param(schema), "max_tokens": 2048}, + litellm_params={}, + headers={}, + ) + + sent_age: Final = body["output_config"]["format"]["schema"]["properties"]["age"] + assert sent_age["type"] == "integer" + assert "minimum" not in sent_age + + BEDROCK_HAIKU_45_BASE_ID: Final = "anthropic.claude-haiku-4-5-20251001-v1:0" @@ -2033,7 +2138,9 @@ def test_bedrock_invoke_claude_default_max_tokens_comes_from_base_model(local_mo headers={}, ) - assert body["max_tokens"] == mapped_cap, f"default max_tokens should be the base model's cap, got {body['max_tokens']}" + assert body["max_tokens"] == mapped_cap, ( + f"default max_tokens should be the base model's cap, got {body['max_tokens']}" + ) def test_bedrock_invoke_claude_explicit_max_tokens_is_kept(local_model_cost_map): @@ -2093,7 +2200,9 @@ def test_bedrock_invoke_claude_default_max_tokens_yields_to_anthropic_config(loc headers={}, ) - assert body["max_tokens"] == configured_max_tokens, f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}" + assert body["max_tokens"] == configured_max_tokens, ( + f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}" + ) @pytest.mark.parametrize(("configured_max_tokens", "deployment_cap"), [(100000, 50), (1000, 8192)]) diff --git a/tests/unit/llms/bedrock/chat/mantle/test_bedrock_chat_mantle_transformation.py b/tests/unit/llms/bedrock/chat/mantle/test_bedrock_chat_mantle_transformation.py index 56d6b039788..154a102fe34 100644 --- a/tests/unit/llms/bedrock/chat/mantle/test_bedrock_chat_mantle_transformation.py +++ b/tests/unit/llms/bedrock/chat/mantle/test_bedrock_chat_mantle_transformation.py @@ -3,6 +3,7 @@ import json import uuid from types import SimpleNamespace from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -246,6 +247,39 @@ async def test_mantle_claude_async_completion_streams_a_wrapped_json_schema_answ assert _streamed_answer(chunks) == BARE_STREAMED_ANSWER +def _json_tool_message(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "anthropic.claude-opus-5-5", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": RISK_ANSWER}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 9}, + }, + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", ("bedrock_mantle/anthropic.claude-opus-5-5", "bedrock/mantle/anthropic.claude-opus-5-5")) +def test_mantle_claude_that_cannot_force_a_tool_keeps_json_schema_on_the_json_tool(model: str) -> None: + respond: Final = MagicMock(wraps=_json_tool_message) + + response: Final = litellm.completion( + **{**_json_schema_stream_kwargs(model), "stream": False}, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + ) + + sent: Final = tuple(json.loads(call.args[0].content) for call in respond.call_args_list) + assert json.loads(response.choices[0].message.content) == RISK_ANSWER + assert [(body.get("output_config", {}).get("format"), [tool["name"] for tool in body["tools"]]) for body in sent] == [ + (None, ["json_tool_call"]) + ] + + def test_mantle_claude_default_max_tokens_comes_from_the_mantle_row(local_model_cost_map): from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 44d3427be91..8cab1018d25 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -4,7 +4,7 @@ import json import logging import os import re -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_tools_pt, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.anthropic.chat.transformation import ( NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES, NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, @@ -31,6 +32,7 @@ from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTI from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject from litellm.types.llms.openai import AllMessageValues +from tests.unit.conftest import NativeStructuredOutputKey def test_transform_usage(): @@ -4981,7 +4983,10 @@ ADAPTIVE_THINKING: Final = {"type": "adaptive"} def _converse_request_for_json_schema( - model: str, stream: bool, thinking_params: Mapping[str, object] | None = None + model: str, + stream: bool, + thinking_params: Mapping[str, object] | None = None, + model_info: Mapping[str, object] | None = None, ) -> tuple[Mapping[str, object], RequestObject]: config = AmazonConverseConfig() optional_params = config.map_openai_params( @@ -4989,12 +4994,13 @@ def _converse_request_for_json_schema( optional_params={}, model=model, drop_params=False, + model_info=model_info, ) request = config._transform_request( model=model, messages=[{"role": "user", "content": "Name the capital of France."}], optional_params=optional_params, - litellm_params={}, + litellm_params={} if model_info is None else {"model_info": model_info}, headers={}, ) return optional_params, request @@ -5061,6 +5067,115 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( assert ("toolConfig" in request) is not native +_FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}} + + +@pytest.mark.parametrize( + ("forced_tool_use", "map_flag", "model_info", "thinking_params", "native", "tool_choice"), + [ + (False, None, None, None, True, None), + (False, None, None, {"thinking": {"type": "adaptive"}}, False, None), + (False, True, None, {"thinking": {"type": "adaptive"}}, True, None), + (False, False, None, None, False, None), + (False, None, {"supports_bedrock_native_structured_output": False}, None, False, None), + (True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE), + ], + ids=[ + "cannot-force-tool", + "cannot-force-tool-thinking", + "cannot-force-tool-thinking-key-true", + "cannot-force-tool-key-false", + "cannot-force-tool-deployment-opt-out", + "can-force-tool", + ], +) +def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out( + set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None], + monkeypatch: pytest.MonkeyPatch, + forced_tool_use: bool, + map_flag: bool | None, + model_info: Mapping[str, object] | None, + thinking_params: Mapping[str, object] | None, + native: bool, + tool_choice: Mapping[str, object] | None, +) -> None: + model: Final = "us.anthropic.claude-haiku-5-5" + monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use) + set_native_structured_output(model, "bedrock" if map_flag else None) + if map_flag is None: + monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output") + + _, request = _converse_request_for_json_schema(model, False, thinking_params, model_info=model_info) + + assert ("outputConfig" in request) is native + assert ("toolConfig" in request) is not native + assert request.get("toolConfig", {}).get("toolChoice") == tool_choice + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params", "native"), + [ + ("us.anthropic.claude-opus-5-5", None, True), + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False), + ("global.anthropic.claude-fable-5-1", None, True), + ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False), + ("us-gov.anthropic.claude-opus-5-5", None, False), + ("us-gov.anthropic.claude-fable-5-1", None, False), + ("us-gov-west-1/anthropic.claude-opus-5-5", None, False), + ("converse/us-gov-east-1/anthropic.claude-fable-5-1", None, False), + ("us-east-1/anthropic.claude-opus-5-5", None, True), + ], +) +def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking( + model: str, thinking_params: Mapping[str, object] | None, native: bool +) -> None: + _, request = _converse_request_for_json_schema(model, False, thinking_params) + + assert ("outputConfig" in request) is native + assert ("toolConfig" in request) is not native + + +def _converse_capital_answer(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": '{"city":"Paris","country":"France"}'}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 12, "totalTokens": 13}, + }, + request=request, + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"] +) +def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming( + model: str, +) -> None: + respond: Final = MagicMock(wraps=_converse_capital_answer) + + stream: Final = litellm.completion( + model=f"bedrock/{model}", + messages=[{"role": "user", "content": "Name the capital of France."}], + response_format=CAPITAL_RESPONSE_FORMAT, + stream=True, + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="fake-secret", + aws_region_name="us-east-1", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + ) + + assert isinstance(stream, CustomStreamWrapper) + streamed: Final = "".join(chunk.choices[0].delta.content or "" for chunk in stream) + sent: Final = respond.call_args.args[0] + assert sent.url.path.endswith("/converse") + assert "textFormat" in json.loads(sent.content)["outputConfig"] + assert json.loads(streamed) == {"city": "Paris", "country": "France"} + + def test_adaptive_thinking_is_kept_next_to_json_schema_on_a_native_output_profile(set_native_structured_output): model: Final = "us.anthropic.claude-opus-5" set_native_structured_output(model, "bedrock") diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 0bd480dcdbc..07fabbf2393 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4175,6 +4175,43 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati assert len(bedrock.bodies) == 1 +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "model_info"), + [ + ("us.anthropic.claude-opus-5-5", None), + ("global.anthropic.claude-fable-5-1", None), + ("us.anthropic.claude-opus-4-8", None), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}), + ], + ids=["opus_5_5_inlined", "fable_5_1_inlined", "mapped_opt_in", "deployment_opt_in"], +) +def test_bedrock_messages_offers_a_native_output_retry_only_when_it_sends_native_output( + model: str, model_info: Mapping[str, object] | None +) -> None: + from litellm.types.router import GenericLiteLLMParams + + config: Final = AmazonAnthropicClaudeMessagesConfig() + litellm_params: Final = GenericLiteLLMParams(model_info=model_info) + optional_params: Final = { + "max_tokens": 100, + "output_config": {"format": {"type": "json_schema", "schema": _CONSTRAINED_SCHEMA}}, + } + + body: Final = REASONING_BODY_ADAPTER.validate_python( + config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + ) + retry_params: Final = config.native_output_fallback_litellm_params(model, optional_params, litellm_params) + + assert (retry_params is not None) is ("format" in REASONING_BODY_ADAPTER.validate_python(body.get("output_config", {}))) + + # Bedrock's 400 bodies for requests past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-29 _STRICT_TOOLS_REJECTIONS: Final = ( {