mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(bedrock): send native structured output for json_schema on Claude models that cannot force a tool (#49703)
* fix(bedrock): send native structured output for json_schema on Claude models that cannot force a tool Opus 5.5 and Sonnet 5.5 on Bedrock cannot force the json tool, so a json_schema request without thinking sent an unforced json tool the model ignored. The opt-in native output key now also applies when the model map says the model cannot force a tool, and the Opus 5.5 commercial Bedrock rows carry the opt-in key. The GovCloud Opus 5.5 rows carry it as false, like the Sonnet 5.5 GovCloud rows * fix(bedrock): keep json tool with thinking on Opus 5.5 and Fable 5.1, and GovCloud on the tool Native output without thinking now applies to every Bedrock Claude model that cannot force the json tool unless supports_bedrock_native_structured_output is false, so Opus 5.5 needs no opt-in row and keeps the json tool when thinking is requested, as #46870 chose. The invoke chat path keeps the native output_format it built instead of inlining it, and GovCloud region-path ids read their own false rows. * fix(bedrock): clean caller output_format schemas before sending them natively on invoke * fix(bedrock): keep Mantle Claude on the json tool and send a caller's output_config.format natively on invoke * fix(bedrock): keep the faked Converse stream for native output that replaces the json tool A json_schema stream that goes native only because the model can't force the json tool now keeps the json tool's faked stream, so it still needs only bedrock:InvokeModel. /v1/messages builds a native-output retry only where it sends native output, and the Mantle test drops its mutable capture. * test(bedrock): cover native output and the opt-out in the invoke json-mode refusal tests --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
183212e8be
commit
94b5b6386a
13 changed files with 518 additions and 45 deletions
|
|
@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
|||
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
|
||||
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_chat_opts_into_native_structured_output,
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_model_is_unknown,
|
||||
|
|
@ -934,8 +935,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
|
||||
|
||||
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
|
||||
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
|
||||
request asks for thinking, which the json tool path would drop.
|
||||
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the
|
||||
request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot
|
||||
force the json tool gets it unless that key is false.
|
||||
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
|
||||
"""
|
||||
return bedrock_chat_uses_native_structured_output(
|
||||
|
|
@ -1316,19 +1318,15 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if "type" in value and value["type"] == "text":
|
||||
return optional_params
|
||||
|
||||
thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params))
|
||||
native_schema: Final = (
|
||||
self._native_output_schema(
|
||||
model,
|
||||
json_schema,
|
||||
self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
|
||||
model_info,
|
||||
)
|
||||
self._native_output_schema(model, json_schema, thinking_requested, model_info)
|
||||
if json_schema is not None
|
||||
else None
|
||||
)
|
||||
if native_schema is not None:
|
||||
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
|
||||
# No synthetic tool injection, no fake_stream needed.
|
||||
# No synthetic tool injection.
|
||||
# Requires an explicit schema — json_object with no schema falls through
|
||||
# to the tool-call path below.
|
||||
output_config: Final = self._create_output_config_for_response_format(
|
||||
|
|
@ -1354,8 +1352,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
optional_params["tool_choice"] = ToolChoiceValuesBlock(
|
||||
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
|
||||
)
|
||||
if non_default_params.get("stream", False) is True:
|
||||
optional_params["fake_stream"] = True
|
||||
# else: response_format=json_object with no schema.
|
||||
# Don't inject the synthetic json_tool_call tool here. When no
|
||||
# schema is given, _create_json_tool_call_for_response_format
|
||||
|
|
@ -1363,6 +1359,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# returns {} instead of the requested JSON. The model already
|
||||
# returns JSON when the prompt asks for it.
|
||||
|
||||
# A request that goes native only because the model can't force the json tool keeps the json tool's faked
|
||||
# stream, so it still needs only bedrock:InvokeModel.
|
||||
streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output(
|
||||
model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info
|
||||
)
|
||||
if json_schema is not None and not streams_natively and non_default_params.get("stream", False) is True:
|
||||
optional_params["fake_stream"] = True
|
||||
optional_params["json_mode"] = True
|
||||
return optional_params
|
||||
|
||||
|
|
|
|||
|
|
@ -129,10 +129,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
json_schema: Mapping[str, object] | None,
|
||||
request_params: Mapping[str, object],
|
||||
) -> bool:
|
||||
return bedrock_chat_uses_native_structured_output(
|
||||
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
|
||||
return self.chat_uses_native_structured_output(
|
||||
model, self.is_thinking_requested(request_params), self._deployment_model_info
|
||||
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
|
||||
|
||||
def chat_uses_native_structured_output(
|
||||
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
|
||||
) -> bool:
|
||||
return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info)
|
||||
|
||||
def uses_opt_in_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> bool:
|
||||
|
|
@ -335,10 +340,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
|
||||
anthropic_request.pop("model", None)
|
||||
anthropic_request.pop("stream", None)
|
||||
model_info: Final = deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params))
|
||||
apply_bedrock_invoke_structured_output(
|
||||
model=model,
|
||||
request_body=anthropic_request,
|
||||
model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
|
||||
model_info=model_info,
|
||||
chat_uses_native_output=lambda: self.chat_uses_native_structured_output(
|
||||
model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info
|
||||
),
|
||||
)
|
||||
strip_unsupported_bedrock_invoke_output_config_keys(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm.llms.anthropic.chat.transformation import JsonModeRequest
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import build_mantle_messages_url
|
||||
from litellm.llms.bedrock.common_utils import bedrock_chat_opts_into_native_structured_output, build_mantle_messages_url
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -54,6 +54,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
|
|||
region=region,
|
||||
)
|
||||
|
||||
def chat_uses_native_structured_output(
|
||||
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
|
||||
) -> bool:
|
||||
return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info)
|
||||
|
||||
def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params))
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import json
|
|||
import os
|
||||
import re
|
||||
import warnings
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
|
||||
|
|
@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
|||
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -315,7 +316,33 @@ def bedrock_supports_native_structured_output(
|
|||
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
|
||||
|
||||
def bedrock_chat_uses_native_structured_output(
|
||||
def _bedrock_map_native_output_flag(model: str) -> object:
|
||||
routed: Final = strip_bedrock_routing_prefix(model)
|
||||
flags: Final = (
|
||||
(entry or {}).get("supports_bedrock_native_structured_output")
|
||||
for entry in (
|
||||
_model_cost_entry(f"bedrock/{routed}"),
|
||||
_model_cost_entry(routed),
|
||||
_model_cost_entry(split_bedrock_region_path(model)[1]),
|
||||
)
|
||||
)
|
||||
return next((flag for flag in flags if isinstance(flag, bool)), None)
|
||||
|
||||
|
||||
def _bedrock_native_output_opted_out(model: str, model_info: Mapping[str, object] | None) -> bool:
|
||||
deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
|
||||
if isinstance(deployment_flag, bool):
|
||||
return not deployment_flag
|
||||
return _bedrock_map_native_output_flag(model) is False
|
||||
|
||||
|
||||
def _bedrock_native_output_replaces_unforceable_tool(model: str, model_info: Mapping[str, object] | None) -> bool:
|
||||
return AnthropicModelInfo.forced_tool_use_unsupported(model) and not _bedrock_native_output_opted_out(
|
||||
model, model_info
|
||||
)
|
||||
|
||||
|
||||
def bedrock_chat_opts_into_native_structured_output(
|
||||
model: str,
|
||||
thinking_requested: bool,
|
||||
custom_llm_provider: str = "bedrock",
|
||||
|
|
@ -326,7 +353,18 @@ def bedrock_chat_uses_native_structured_output(
|
|||
)
|
||||
|
||||
|
||||
def bedrock_native_output_is_opt_in(
|
||||
def bedrock_chat_uses_native_structured_output(
|
||||
model: str,
|
||||
thinking_requested: bool,
|
||||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
return bedrock_chat_opts_into_native_structured_output(
|
||||
model, thinking_requested, custom_llm_provider, model_info
|
||||
) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info))
|
||||
|
||||
|
||||
def bedrock_native_output_is_deployment_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(
|
||||
|
|
@ -334,6 +372,15 @@ def bedrock_native_output_is_opt_in(
|
|||
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
|
||||
|
||||
def bedrock_native_output_is_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
|
||||
_bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
|
||||
)
|
||||
|
||||
|
||||
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
**litellm_params,
|
||||
|
|
@ -424,13 +471,18 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
|
|||
return False
|
||||
|
||||
|
||||
def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
|
||||
return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
|
||||
anthropic_optional_params
|
||||
)
|
||||
|
||||
|
||||
def requests_opt_in_native_output(
|
||||
model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
|
||||
) -> bool:
|
||||
native_output_requested: Final = isinstance(
|
||||
anthropic_optional_params.get("output_format"), Mapping
|
||||
) or requests_output_config_format(anthropic_optional_params)
|
||||
return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
|
||||
return requests_native_output(anthropic_optional_params) and bedrock_native_output_is_opt_in(
|
||||
model, model_info=model_info
|
||||
)
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -491,6 +543,7 @@ def apply_bedrock_invoke_structured_output(
|
|||
model: str,
|
||||
request_body: dict[str, object], # mutable-ok: edited in place like siblings
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
chat_uses_native_output: Callable[[], bool] = lambda: False,
|
||||
) -> None:
|
||||
"""
|
||||
Route Anthropic structured-output params to what the Bedrock model supports.
|
||||
|
|
@ -502,7 +555,8 @@ def apply_bedrock_invoke_structured_output(
|
|||
forwarded as ``output_config.format``, which Bedrock relays to the model for
|
||||
enforced structured output. For every other model the schema is inlined into
|
||||
the last user message as best-effort text, with a warning because nothing
|
||||
enforces it.
|
||||
enforces it. ``chat_uses_native_output`` is asked only when a schema is
|
||||
present, and a true answer sends it natively after the same cleanup.
|
||||
"""
|
||||
legacy_output_format: Final = request_body.pop("output_format", None)
|
||||
output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
|
||||
|
|
@ -511,7 +565,15 @@ def apply_bedrock_invoke_structured_output(
|
|||
if schema_format is None:
|
||||
return
|
||||
|
||||
native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
|
||||
native_format: Final = bedrock_native_output_format(
|
||||
model,
|
||||
schema_format,
|
||||
model_info=(
|
||||
{**(model_info or {}), "supports_bedrock_native_structured_output": True}
|
||||
if chat_uses_native_output()
|
||||
else model_info
|
||||
),
|
||||
)
|
||||
if native_format is not None:
|
||||
existing_output_config: Final = request_body.get("output_config")
|
||||
if isinstance(existing_output_config, dict):
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
|
||||
BedrockError,
|
||||
apply_bedrock_invoke_structured_output,
|
||||
bedrock_native_output_is_deployment_opt_in,
|
||||
bedrock_supports_tool_search,
|
||||
ensure_bedrock_anthropic_messages_tool_names,
|
||||
get_anthropic_beta_from_headers,
|
||||
|
|
@ -47,7 +48,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_bedrock_opus_output_config_effort,
|
||||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
requests_opt_in_native_output,
|
||||
requests_native_output,
|
||||
requests_output_config_format,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
supports_bedrock_prompt_cache_ttl,
|
||||
|
|
@ -131,7 +132,10 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
litellm_params: GenericLiteLLMParams,
|
||||
) -> GenericLiteLLMParams | None:
|
||||
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
|
||||
if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
|
||||
if not (
|
||||
requests_native_output(anthropic_messages_optional_request_params)
|
||||
and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
|
||||
):
|
||||
return None
|
||||
return litellm_params.model_copy(
|
||||
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
|
||||
|
|
|
|||
|
|
@ -49635,6 +49635,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -49679,6 +49680,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -68344,6 +68346,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -68389,6 +68392,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -68617,6 +68621,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -68657,6 +68662,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
|
|||
|
|
@ -49635,6 +49635,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -49679,6 +49680,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -68344,6 +68346,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -68389,6 +68392,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -68617,6 +68621,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -68657,6 +68662,7 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_bedrock_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
|
|||
|
|
@ -201,20 +201,27 @@ def _marker() -> str:
|
|||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def _bedrock_invoke_deployment(
|
||||
scenario: Scenario, wire_url: str, model_info: Mapping[str, JsonValue] | None = None
|
||||
) -> str:
|
||||
return scenario.model(
|
||||
model=f"bedrock/invoke/{BEDROCK_MODEL}",
|
||||
model_info=model_info,
|
||||
api_base=wire_url,
|
||||
aws_access_key_id="AKIASCRIPTEDPROVIDER",
|
||||
aws_secret_access_key="scripted-secret",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire_url,
|
||||
)
|
||||
|
||||
|
||||
def _deployment(scenario: Scenario, provider: Provider, rig: Rig) -> str:
|
||||
wire_url: Final = rig.wire.url
|
||||
match provider:
|
||||
case "anthropic":
|
||||
return scenario.model(model=f"anthropic/{MODEL}", api_base=wire_url, api_key=_ANTHROPIC_KEY)
|
||||
case "bedrock_invoke":
|
||||
return scenario.model(
|
||||
model=f"bedrock/invoke/{BEDROCK_MODEL}",
|
||||
api_base=wire_url,
|
||||
aws_access_key_id="AKIASCRIPTEDPROVIDER",
|
||||
aws_secret_access_key="scripted-secret",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire_url,
|
||||
)
|
||||
return _bedrock_invoke_deployment(scenario, wire_url)
|
||||
case "claude_platform":
|
||||
return scenario.model(
|
||||
model=f"bedrock/claude_platform/{MODEL}",
|
||||
|
|
@ -1184,7 +1191,7 @@ def test_stream_cache_hit_skips_cached_provider_fields_that_are_not_an_object(ri
|
|||
assert tuple(_message_fields(event["response"]) for event in stale_logs) == ((),) * len(stale_logs), stale_logs
|
||||
|
||||
|
||||
JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke"]
|
||||
JsonModeRoute: TypeAlias = Literal["anthropic_native", "anthropic_tool", "bedrock_invoke_native", "bedrock_invoke_tool"]
|
||||
_UNMAPPED_MODEL: Final = "claude-scripted-9"
|
||||
_JSON_ANSWER: Final = json.dumps({"answer": DOCS_PARTIAL_TEXT})
|
||||
|
||||
|
|
@ -1216,11 +1223,20 @@ _JSON_MODE_CASES: Final[Mapping[str, JsonModeCase]] = {
|
|||
"anthropic_tool_enveloped": JsonModeCase(
|
||||
"anthropic_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
|
||||
),
|
||||
"bedrock_invoke_native": JsonModeCase(
|
||||
"bedrock_invoke_native",
|
||||
"partial",
|
||||
DOCS_PARTIAL_TEXT,
|
||||
DOCS_PARTIAL_TEXT,
|
||||
"content_filter",
|
||||
"content_filter",
|
||||
"native",
|
||||
),
|
||||
"bedrock_invoke_declared": JsonModeCase(
|
||||
"bedrock_invoke", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
|
||||
"bedrock_invoke_tool", "jsontool", DECLARED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
|
||||
),
|
||||
"bedrock_invoke_enveloped": JsonModeCase(
|
||||
"bedrock_invoke", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
|
||||
"bedrock_invoke_tool", "jsonheld", ENVELOPED_PARTIAL_JSON, _JSON_ANSWER, "content_filter", "stop", "json_tool"
|
||||
),
|
||||
}
|
||||
|
||||
|
|
@ -1234,8 +1250,15 @@ def _json_mode_deployment(scenario: Scenario, route: JsonModeRoute, rig: Rig) ->
|
|||
scenario.model(model=f"anthropic/{_UNMAPPED_MODEL}", api_base=rig.wire.url, api_key=_ANTHROPIC_KEY),
|
||||
"anthropic",
|
||||
)
|
||||
case "bedrock_invoke":
|
||||
case "bedrock_invoke_native":
|
||||
return _deployment(scenario, "bedrock_invoke", rig), "bedrock_invoke"
|
||||
case "bedrock_invoke_tool":
|
||||
return (
|
||||
_bedrock_invoke_deployment(
|
||||
scenario, rig.wire.url, {"supports_bedrock_native_structured_output": False}
|
||||
),
|
||||
"bedrock_invoke",
|
||||
)
|
||||
|
||||
|
||||
def _requested_format(body: Mapping[str, JsonValue]) -> tuple[bool, bool]:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ _FABLE: Final = "us.anthropic.claude-fable-5"
|
|||
_SONNET: Final = "us.anthropic.claude-sonnet-4-6"
|
||||
_OPUS: Final = "us.anthropic.claude-opus-5-5"
|
||||
_OPT_IN: Final = _OPUS
|
||||
_UNFORCEABLE_TOOL: Final = _OPUS
|
||||
_OPUS_4_8: Final = "us.anthropic.claude-opus-4-8"
|
||||
_DEEPSEEK: Final = "deepseek.v3.2"
|
||||
_AWS: Final[dict[str, JsonValue]] = {
|
||||
|
|
@ -536,6 +537,12 @@ def _resolves(schema: Mapping[str, JsonValue], ref: str) -> bool:
|
|||
return ref.startswith(prefix) and ref.removeprefix(prefix) in object_value(schema.get("$defs") or {})
|
||||
|
||||
|
||||
def _tool_choice(body: Mapping[str, JsonValue], invoke: bool) -> JsonValue:
|
||||
if invoke:
|
||||
return body.get("tool_choice")
|
||||
return object_value(body.get("toolConfig") or {}).get("toolChoice")
|
||||
|
||||
|
||||
def _assert_native_without_tools(body: Mapping[str, JsonValue], invoke: bool, schema: Mapping[str, JsonValue]) -> None:
|
||||
assert _native_schema(body, invoke) == schema, body
|
||||
assert _JSON_TOOL not in _tools(body, invoke), body
|
||||
|
|
@ -631,6 +638,40 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa
|
|||
assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["converse", "invoke"])
|
||||
def test_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool_gets_native_output(
|
||||
gateway: Gateway, route: str
|
||||
) -> None:
|
||||
marker, marker_hex = _marker()
|
||||
with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA))
|
||||
)
|
||||
assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text
|
||||
_, body = _one_call(wire, marker_hex)
|
||||
_assert_native_without_tools(body, route == "invoke", _SCHEMA)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["converse", "invoke"])
|
||||
def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_output_with_an_unforced_json_tool(
|
||||
gateway: Gateway, route: str
|
||||
) -> None:
|
||||
marker, marker_hex = _marker()
|
||||
invoke: Final = route == "invoke"
|
||||
with wire_server(_rejecting_native_output(_FABLE_REJECTION)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, wire, route, _UNFORCEABLE_TOOL)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA))
|
||||
)
|
||||
assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text
|
||||
rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain())
|
||||
assert _native_schema(rejected, invoke) == _SCHEMA, rejected
|
||||
assert _native_schema(retried, invoke) is None, retried
|
||||
assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried
|
||||
assert _tool_choice(retried, invoke) is None, retried
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "model_info", "native"),
|
||||
[
|
||||
|
|
@ -797,6 +838,25 @@ def test_thinking_with_a_caller_tool_on_a_bedrock_native_model_sends_only_the_ca
|
|||
assert _thinking_requested(body, False), body
|
||||
|
||||
|
||||
def test_a_caller_tool_next_to_json_schema_without_thinking_on_a_model_that_cannot_force_the_json_tool(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker, marker_hex = _marker()
|
||||
with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, wire, "converse", _UNFORCEABLE_TOOL)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
_chat_body(model, marker, tools=[_USER_TOOL_SPEC], response_format=_response_format(_SCHEMA)),
|
||||
)
|
||||
(call,) = list_value(_chat_message(response)["tool_calls"])
|
||||
function: Final = object_value(object_value(call)["function"])
|
||||
assert function["name"] == _USER_TOOL, response.text
|
||||
_, body = _one_call(wire, marker_hex)
|
||||
assert _tools(body, False).keys() == {_USER_TOOL}, body
|
||||
assert _native_schema(body, False) == _SCHEMA, body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "expected"),
|
||||
[(_UNCLEANED_SCHEMA, _UNCLEANED_SCHEMA), (_NULLABLE_SCHEMA, _CLOSED_NULLABLE_SCHEMA)],
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import copy
|
|||
import json
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -20,6 +20,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transform
|
|||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from tests.unit.conftest import NativeStructuredOutputKey
|
||||
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
|
||||
|
||||
ONE_PIXEL_PNG = base64.b64decode(
|
||||
|
|
@ -636,7 +637,6 @@ def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_nat
|
|||
assert "tools" not in result
|
||||
|
||||
|
||||
|
||||
class Relabel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
@ -673,6 +673,7 @@ def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_re
|
|||
assert item_ref.removeprefix("#/$defs/") in sent["$defs"]
|
||||
assert "tools" not in result
|
||||
|
||||
|
||||
def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map):
|
||||
"""Bedrock Invoke chat path forwards ``output_config.format`` alongside effort
|
||||
for models with native structured-output support (Claude Sonnet 4.6)."""
|
||||
|
|
@ -1385,6 +1386,7 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
|
|
@ -1394,6 +1396,7 @@ def event_loop():
|
|||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown(event_loop):
|
||||
import litellm
|
||||
|
|
@ -1427,6 +1430,7 @@ def setup_and_teardown(event_loop):
|
|||
if pending:
|
||||
event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
|
||||
|
||||
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"set_verbose": getattr(litellm, "set_verbose", False),
|
||||
|
|
@ -1498,6 +1502,7 @@ IN WITNESS WHEREOF, the parties have executed this Agreement.
|
|||
* 8
|
||||
) # Repeat to ensure we have enough tokens (need 1024+ for Claude models)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockAnthropicPromptCachingRegression:
|
||||
"""
|
||||
|
|
@ -1660,6 +1665,7 @@ class TestBedrockAnthropicPromptCachingRegression:
|
|||
if "anthropic_beta" in additional_fields:
|
||||
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockAnthropic1MContextRegression:
|
||||
"""
|
||||
|
|
@ -1845,6 +1851,7 @@ class TestBedrockAnthropic1MContextRegression:
|
|||
assert "context-1m-2025-08-07" in beta_headers
|
||||
assert "computer-use-2024-10-22" in beta_headers
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockAnthropicCombinedRegressions:
|
||||
"""
|
||||
|
|
@ -2011,6 +2018,104 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"),
|
||||
[
|
||||
(False, None, None, {}, True),
|
||||
(False, None, None, {"thinking": {"type": "adaptive"}}, False),
|
||||
(False, True, None, {"thinking": {"type": "adaptive"}}, True),
|
||||
(False, False, None, {}, False),
|
||||
(False, None, {"supports_bedrock_native_structured_output": False}, {}, False),
|
||||
(True, True, None, {}, False),
|
||||
],
|
||||
ids=[
|
||||
"cannot-force-tool",
|
||||
"cannot-force-tool-thinking",
|
||||
"cannot-force-tool-thinking-key-true",
|
||||
"cannot-force-tool-key-false",
|
||||
"cannot-force-tool-deployment-opt-out",
|
||||
"can-force-tool",
|
||||
],
|
||||
)
|
||||
def test_invoke_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out(
|
||||
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
forced_tool_use: bool,
|
||||
map_flag: bool | None,
|
||||
model_info: Mapping[str, object] | None,
|
||||
thinking_params: Mapping[str, object],
|
||||
native: bool,
|
||||
) -> None:
|
||||
model: Final = "us.anthropic.claude-haiku-5-5"
|
||||
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
|
||||
set_native_structured_output(model, "bedrock" if map_flag else None)
|
||||
if map_flag is None:
|
||||
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
|
||||
|
||||
config: Final = AmazonAnthropicClaudeConfig(model_info=model_info)
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"response_format": _CITY_RESPONSE_FORMAT, "max_tokens": 2048, **thinking_params},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
body: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Capital of France?"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={} if model_info is None else {"model_info": model_info},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert (body.get("output_config", {}).get("format", {}).get("schema") == _CITY_SCHEMA) is native
|
||||
assert ("tools" in body) is not native
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"caller_schema_param",
|
||||
[
|
||||
lambda schema: {"output_format": {"type": "json_schema", "schema": schema}},
|
||||
lambda schema: {"output_config": {"format": {"type": "json_schema", "schema": schema}}},
|
||||
],
|
||||
ids=["output_format", "output_config.format"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "forced_tool_use", "native_key"),
|
||||
[("us.anthropic.claude-opus-4-8", True, "legacy"), ("us.anthropic.claude-haiku-5-5", False, None)],
|
||||
ids=["native-key", "cannot-force-tool"],
|
||||
)
|
||||
def test_invoke_caller_schema_drops_unsupported_constraints_before_going_native(
|
||||
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
forced_tool_use: bool,
|
||||
native_key: NativeStructuredOutputKey,
|
||||
caller_schema_param: Callable[[Mapping[str, object]], dict[str, object]],
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
|
||||
set_native_structured_output(model, native_key)
|
||||
if native_key is None:
|
||||
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"age": {"type": "integer", "minimum": 0}},
|
||||
"required": ["age"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
body: Final = AmazonAnthropicClaudeConfig().transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "How old is the oldest tree?"}],
|
||||
optional_params={**caller_schema_param(schema), "max_tokens": 2048},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
sent_age: Final = body["output_config"]["format"]["schema"]["properties"]["age"]
|
||||
assert sent_age["type"] == "integer"
|
||||
assert "minimum" not in sent_age
|
||||
|
||||
|
||||
BEDROCK_HAIKU_45_BASE_ID: Final = "anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
|
||||
|
||||
|
|
@ -2033,7 +2138,9 @@ def test_bedrock_invoke_claude_default_max_tokens_comes_from_base_model(local_mo
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert body["max_tokens"] == mapped_cap, f"default max_tokens should be the base model's cap, got {body['max_tokens']}"
|
||||
assert body["max_tokens"] == mapped_cap, (
|
||||
f"default max_tokens should be the base model's cap, got {body['max_tokens']}"
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_invoke_claude_explicit_max_tokens_is_kept(local_model_cost_map):
|
||||
|
|
@ -2093,7 +2200,9 @@ def test_bedrock_invoke_claude_default_max_tokens_yields_to_anthropic_config(loc
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert body["max_tokens"] == configured_max_tokens, f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}"
|
||||
assert body["max_tokens"] == configured_max_tokens, (
|
||||
f"AnthropicConfig(max_tokens) was ignored, got {body['max_tokens']}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("configured_max_tokens", "deployment_cap"), [(100000, 50), (1000, 8192)])
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import json
|
|||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -246,6 +247,39 @@ async def test_mantle_claude_async_completion_streams_a_wrapped_json_schema_answ
|
|||
assert _streamed_answer(chunks) == BARE_STREAMED_ANSWER
|
||||
|
||||
|
||||
def _json_tool_message(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-opus-5-5",
|
||||
"content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": RISK_ANSWER}],
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 9},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("model", ("bedrock_mantle/anthropic.claude-opus-5-5", "bedrock/mantle/anthropic.claude-opus-5-5"))
|
||||
def test_mantle_claude_that_cannot_force_a_tool_keeps_json_schema_on_the_json_tool(model: str) -> None:
|
||||
respond: Final = MagicMock(wraps=_json_tool_message)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
**{**_json_schema_stream_kwargs(model), "stream": False},
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
)
|
||||
|
||||
sent: Final = tuple(json.loads(call.args[0].content) for call in respond.call_args_list)
|
||||
assert json.loads(response.choices[0].message.content) == RISK_ANSWER
|
||||
assert [(body.get("output_config", {}).get("format"), [tool["name"] for tool in body["tools"]]) for body in sent] == [
|
||||
(None, ["json_tool_call"])
|
||||
]
|
||||
|
||||
|
||||
def test_mantle_claude_default_max_tokens_comes_from_the_mantle_row(local_model_cost_map):
|
||||
from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
|
|
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
_bedrock_tools_pt,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.anthropic.chat.transformation import (
|
||||
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES,
|
||||
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS,
|
||||
|
|
@ -31,6 +32,7 @@ from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTI
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from tests.unit.conftest import NativeStructuredOutputKey
|
||||
|
||||
|
||||
def test_transform_usage():
|
||||
|
|
@ -4981,7 +4983,10 @@ ADAPTIVE_THINKING: Final = {"type": "adaptive"}
|
|||
|
||||
|
||||
def _converse_request_for_json_schema(
|
||||
model: str, stream: bool, thinking_params: Mapping[str, object] | None = None
|
||||
model: str,
|
||||
stream: bool,
|
||||
thinking_params: Mapping[str, object] | None = None,
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> tuple[Mapping[str, object], RequestObject]:
|
||||
config = AmazonConverseConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
|
|
@ -4989,12 +4994,13 @@ def _converse_request_for_json_schema(
|
|||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
model_info=model_info,
|
||||
)
|
||||
request = config._transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Name the capital of France."}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
litellm_params={} if model_info is None else {"model_info": model_info},
|
||||
headers={},
|
||||
)
|
||||
return optional_params, request
|
||||
|
|
@ -5061,6 +5067,115 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested(
|
|||
assert ("toolConfig" in request) is not native
|
||||
|
||||
|
||||
_FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("forced_tool_use", "map_flag", "model_info", "thinking_params", "native", "tool_choice"),
|
||||
[
|
||||
(False, None, None, None, True, None),
|
||||
(False, None, None, {"thinking": {"type": "adaptive"}}, False, None),
|
||||
(False, True, None, {"thinking": {"type": "adaptive"}}, True, None),
|
||||
(False, False, None, None, False, None),
|
||||
(False, None, {"supports_bedrock_native_structured_output": False}, None, False, None),
|
||||
(True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE),
|
||||
],
|
||||
ids=[
|
||||
"cannot-force-tool",
|
||||
"cannot-force-tool-thinking",
|
||||
"cannot-force-tool-thinking-key-true",
|
||||
"cannot-force-tool-key-false",
|
||||
"cannot-force-tool-deployment-opt-out",
|
||||
"can-force-tool",
|
||||
],
|
||||
)
|
||||
def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thinking_unless_opted_out(
|
||||
set_native_structured_output: Callable[[str, NativeStructuredOutputKey], None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
forced_tool_use: bool,
|
||||
map_flag: bool | None,
|
||||
model_info: Mapping[str, object] | None,
|
||||
thinking_params: Mapping[str, object] | None,
|
||||
native: bool,
|
||||
tool_choice: Mapping[str, object] | None,
|
||||
) -> None:
|
||||
model: Final = "us.anthropic.claude-haiku-5-5"
|
||||
monkeypatch.setitem(litellm.model_cost[model], "supports_forced_tool_use", forced_tool_use)
|
||||
set_native_structured_output(model, "bedrock" if map_flag else None)
|
||||
if map_flag is None:
|
||||
monkeypatch.delitem(litellm.model_cost[model], "supports_bedrock_native_structured_output")
|
||||
|
||||
_, request = _converse_request_for_json_schema(model, False, thinking_params, model_info=model_info)
|
||||
|
||||
assert ("outputConfig" in request) is native
|
||||
assert ("toolConfig" in request) is not native
|
||||
assert request.get("toolConfig", {}).get("toolChoice") == tool_choice
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "thinking_params", "native"),
|
||||
[
|
||||
("us.anthropic.claude-opus-5-5", None, True),
|
||||
("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False),
|
||||
("global.anthropic.claude-fable-5-1", None, True),
|
||||
("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False),
|
||||
("us-gov.anthropic.claude-opus-5-5", None, False),
|
||||
("us-gov.anthropic.claude-fable-5-1", None, False),
|
||||
("us-gov-west-1/anthropic.claude-opus-5-5", None, False),
|
||||
("converse/us-gov-east-1/anthropic.claude-fable-5-1", None, False),
|
||||
("us-east-1/anthropic.claude-opus-5-5", None, True),
|
||||
],
|
||||
)
|
||||
def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking(
|
||||
model: str, thinking_params: Mapping[str, object] | None, native: bool
|
||||
) -> None:
|
||||
_, request = _converse_request_for_json_schema(model, False, thinking_params)
|
||||
|
||||
assert ("outputConfig" in request) is native
|
||||
assert ("toolConfig" in request) is not native
|
||||
|
||||
|
||||
def _converse_capital_answer(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": '{"city":"Paris","country":"France"}'}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 12, "totalTokens": 13},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"]
|
||||
)
|
||||
def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming(
|
||||
model: str,
|
||||
) -> None:
|
||||
respond: Final = MagicMock(wraps=_converse_capital_answer)
|
||||
|
||||
stream: Final = litellm.completion(
|
||||
model=f"bedrock/{model}",
|
||||
messages=[{"role": "user", "content": "Name the capital of France."}],
|
||||
response_format=CAPITAL_RESPONSE_FORMAT,
|
||||
stream=True,
|
||||
aws_access_key_id="AKIAEXAMPLE",
|
||||
aws_secret_access_key="fake-secret",
|
||||
aws_region_name="us-east-1",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
)
|
||||
|
||||
assert isinstance(stream, CustomStreamWrapper)
|
||||
streamed: Final = "".join(chunk.choices[0].delta.content or "" for chunk in stream)
|
||||
sent: Final = respond.call_args.args[0]
|
||||
assert sent.url.path.endswith("/converse")
|
||||
assert "textFormat" in json.loads(sent.content)["outputConfig"]
|
||||
assert json.loads(streamed) == {"city": "Paris", "country": "France"}
|
||||
|
||||
|
||||
def test_adaptive_thinking_is_kept_next_to_json_schema_on_a_native_output_profile(set_native_structured_output):
|
||||
model: Final = "us.anthropic.claude-opus-5"
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
|
|
|||
|
|
@ -4175,6 +4175,43 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati
|
|||
assert len(bedrock.bodies) == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_info"),
|
||||
[
|
||||
("us.anthropic.claude-opus-5-5", None),
|
||||
("global.anthropic.claude-fable-5-1", None),
|
||||
("us.anthropic.claude-opus-4-8", None),
|
||||
("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}),
|
||||
],
|
||||
ids=["opus_5_5_inlined", "fable_5_1_inlined", "mapped_opt_in", "deployment_opt_in"],
|
||||
)
|
||||
def test_bedrock_messages_offers_a_native_output_retry_only_when_it_sends_native_output(
|
||||
model: str, model_info: Mapping[str, object] | None
|
||||
) -> None:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
config: Final = AmazonAnthropicClaudeMessagesConfig()
|
||||
litellm_params: Final = GenericLiteLLMParams(model_info=model_info)
|
||||
optional_params: Final = {
|
||||
"max_tokens": 100,
|
||||
"output_config": {"format": {"type": "json_schema", "schema": _CONSTRAINED_SCHEMA}},
|
||||
}
|
||||
|
||||
body: Final = REASONING_BODY_ADAPTER.validate_python(
|
||||
config.transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}],
|
||||
anthropic_messages_optional_request_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers={},
|
||||
)
|
||||
)
|
||||
retry_params: Final = config.native_output_fallback_litellm_params(model, optional_params, litellm_params)
|
||||
|
||||
assert (retry_params is not None) is ("format" in REASONING_BODY_ADAPTER.validate_python(body.get("output_config", {})))
|
||||
|
||||
|
||||
# Bedrock's 400 bodies for requests past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-29
|
||||
_STRICT_TOOLS_REJECTIONS: Final = (
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue