diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 044123d8729..b704dfa2675 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -9,7 +9,8 @@ import json import mimetypes import re from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from itertools import groupby, islice +from graphlib import CycleError, TopologicalSorter +from itertools import chain, groupby, islice from os import PathLike from pathlib import Path from types import MappingProxyType @@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, frontier = tuple(child for node in frontier for child in _subschemas(node)) +def schema_fragment_ref(ref: str) -> str: + return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref + + +def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool: + definitions: Final = MappingProxyType(dict(_root_definitions(schema))) + if not _refs_in(schema) <= definitions.keys(): + return True + try: + tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order()) + except CycleError: + return True + return False + + +def schema_union_property_count(schema: Mapping[str, object]) -> int: + return sum(1 for _, node in _property_schemas(schema) if _is_union(node)) + + +def schema_optional_property_count(schema: Mapping[str, object]) -> int: + return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema)) + + +def schema_property_count(schema: Mapping[str, object]) -> int: + return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties"))) + + +def schema_grammar_branch_count(schema: Mapping[str, object]) -> int: + return sum(_grammar_branches(node) for node in _schema_nodes(schema)) + + +def schema_has_open_object(schema: Mapping[str, object]) -> bool: + return any(_is_open_object(node) for node in _schema_nodes(schema)) + + +def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool: + return any( + isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern) + for node in _schema_nodes(schema) + ) + + +def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str) + ) + + +def _may_be_string(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type is None or node_type == "string" or "string" in _list_at(node, "type") + + +def is_object_schema(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) + + +def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING)) + + +_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) + + +def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: + value: Final = container.get(key) + return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () + + +def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]: + if container is None: + return iter(()) + return ((key, value) for key in container if (value := _schema_at(container, key)) is not None) + + +def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + for defs_key in ("$defs", "definitions"): + yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key))) + + +def _refs_in(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str) + ) + + +def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema)) + + +def _is_union(node: Mapping[str, object]) -> bool: + return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1 + + +def _grammar_branches(node: Mapping[str, object]) -> int: + properties: Final = _schema_at(node, "properties") + if properties is None: + return 0 + optional: Final = len(_optional_property_names(node)) + return 2**optional * (len(properties) - optional + 1) + + +def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: + properties: Final = _schema_at(node, "properties") + if properties is None: + return frozenset() + return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str)) + + +def _is_open_object(node: Mapping[str, object]) -> bool: + additional_properties: Final = node.get("additionalProperties") + if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node: + return True + return is_object_schema(node) and "properties" not in node and additional_properties is not False + + def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: for key, value in node.items(): if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 87e76fe39f7..0e9206b422e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5504,14 +5504,14 @@ def _bedrock_tools_pt( description = name defs = parameters.pop("$defs", {}) - defs_copy = copy.deepcopy(defs) + defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {} # Expand $ref references in parameters using the definitions # Note: We don't pre-flatten defs as that causes exponential memory growth # with circular references (see issue #19098). unpack_defs handles nested # refs recursively and correctly detects/skips circular references. unpack_defs(parameters, defs_copy) normalize_json_schema_custom_types_to_object(parameters) - if parameters.get("type") not in _valid_json_schema_root_types: + if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types): parameters["type"] = "object" tool_block = cast( BedrockToolBlock, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0b2f75e352b..38af6e7998b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1,3 +1,4 @@ +import copy import json import re import time @@ -24,7 +25,18 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( + DEFS_MAX_INLINED_BYTES, + is_object_schema, sanitize_input_schema_for_anthropic, + schema_fragment_ref, + schema_grammar_branch_count, + schema_has_lookaround_pattern, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_optional_property_count, + schema_string_formats, + schema_union_property_count, + unpack_defs, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, @@ -43,7 +55,7 @@ from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, unwrap_response_format_envelope, ) -from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, @@ -150,6 +162,12 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128 _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) +_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( + ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") +) +NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16 +NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24 +NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048 def _tool_type(tool: object) -> str | None: @@ -753,7 +771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): grammar Anthropic compiles the schema into, so the API rejects them with a 400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type, property 'uniqueItems' is not supported"): - - maxItems/minItems/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints + - maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints - minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints - minLength/maxLength: string constraints - minProperties/maxProperties/patternProperties/propertyNames: object constraints @@ -810,7 +828,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "else": "conditional (else): {}", "not": "must not match: {}", } - unsupported_fields: Final = set(constraint_labels) + min_items: Final = schema.get("minItems") + enforced_min_items: Final = ( + isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1) + ) + unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set()) # Build description additions from removed constraints. Iterating # constraint_labels (not the set) keeps the note order deterministic across @@ -818,7 +840,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # PYTHONHASHSEED and stay cache-friendly. constraint_descriptions: Final[list] = [] for field, label in constraint_labels.items(): - if field not in schema: + if field not in schema or field not in unsupported_fields: continue value = schema[field] # A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no @@ -853,7 +875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Already handled above continue - if key in ("properties", "$defs") and isinstance(value, dict): + if key in ("properties", "$defs", "definitions") and isinstance(value, dict): nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()} elif key == "items" and isinstance(value, dict): @@ -865,6 +887,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ] elif key == "allOf" and isinstance(value, list): result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value] + elif key == "$ref" and isinstance(value, str): + result[key] = schema_fragment_ref(value) elif key == "oneOf" and isinstance(value, list): # Anthropic rejects oneOf ("Schema type 'oneOf' is not supported"); # the Anthropic SDK rewrites it to anyOf, so do the same. @@ -876,7 +900,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Anthropic requires additionalProperties=false for object schemas # See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs - if result.get("type") == "object": + if is_object_schema(result): result["additionalProperties"] = False return result @@ -1628,32 +1652,55 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return json_schema + @staticmethod + def _defs_unpacked(json_schema: Mapping[str, object]) -> Mapping[str, object]: + unpacked: Final = copy.deepcopy(dict(json_schema)) + defs: Final = unpacked.pop("$defs", unpacked.pop("definitions", None)) + if isinstance(defs, dict) and defs: + unpack_defs(unpacked, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES) + return unpacked + + @staticmethod + def native_output_schema(json_schema: Mapping[str, object]) -> Mapping[str, object] | None: + if ( + schema_has_unresolvable_ref(json_schema) + or schema_has_open_object(json_schema) + or schema_has_lookaround_pattern(json_schema) + or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS + or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS + or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS + or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES + ): + return None + return AnthropicConfig.filter_anthropic_output_schema(dict(json_schema)) + + def uses_native_structured_output_for_schema( + self, + model: str, + json_schema: Mapping[str, object] | None, + request_params: Mapping[str, object], + ) -> bool: + open_object_takes_forced_json_tool: Final = ( + json_schema is not None + and schema_has_open_object(json_schema) + and not self.is_thinking_requested(request_params) + and not self.is_thinking_enabled(dict(request_params)) + and not request_params.get("tools") + and not AnthropicModelInfo.forced_tool_use_unsupported(model) + ) + return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + + def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: + return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) + def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None: - json_schema: dict | None = self._extract_json_schema_from_response_format(value) + json_schema: Final = self._extract_json_schema_from_response_format(value) if json_schema is None: return None - - # Resolve $ref/$defs before filtering — Anthropic doesn't support - # external schema references (e.g., /$defs/CalendarEvent). - import copy - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - DEFS_MAX_INLINED_BYTES, - unpack_defs, - ) - - json_schema = copy.deepcopy(json_schema) - defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {})) - if defs: - unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES) - - # Filter out unsupported fields for Anthropic's output_format API - filtered_schema: Final = self.filter_anthropic_output_schema(json_schema) - - return AnthropicOutputSchema( - type="json_schema", - schema=filtered_schema, - ) + native_schema: Final = self.native_output_format_schema(json_schema) + if native_schema is None: + return None + return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) def map_response_format_to_anthropic_tool( self, value: dict | None, optional_params: dict, is_thinking_enabled: bool @@ -1820,7 +1867,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): output_key=param, ) elif param == "response_format" and isinstance(value, dict): - if self.uses_native_structured_output(model): + if self.uses_native_structured_output_for_schema( + model, + self._extract_json_schema_from_response_format(value), + REQUEST_PARAMS.validate_python(non_default_params), + ): _output_format = self.map_response_format_to_anthropic_output_format(value) if _output_format is not None: optional_params["output_format"] = _output_format @@ -2979,6 +3030,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): completion_response["stop_reason"] = "stop" if refusal_fields: json_mode_message.provider_specific_fields = dict(refusal_fields) + if thinking_blocks: + json_mode_message.thinking_blocks = thinking_blocks + json_mode_message.reasoning_content = reasoning_content _message = json_mode_message model_response.choices[0].message = _message diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index db4d5e0f820..51c8e4b8bef 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Union import httpx -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter from litellm.constants import DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -38,6 +38,9 @@ else: LiteLLMLoggingObj = Any +REQUEST_PARAMS: Final = TypeAdapter(Mapping[str, object]) + + class BaseLLMException(Exception): def __init__( self, @@ -126,6 +129,17 @@ class BaseConfig(ABC): thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None + def is_thinking_requested(self, non_default_params: Mapping[str, object]) -> bool: + match non_default_params: + case {"thinking": True | {"type": "enabled" | "adaptive"}}: + return True + case {"reasoning_effort": None | "none"}: + return False + case {"reasoning_effort": object()}: + return True + case _: + return False + def is_max_tokens_in_request(self, non_default_params: dict) -> bool: """ OpenAI spec allows max_tokens or max_completion_tokens to be specified. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 1f09b3724f1..66ee920afd0 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -29,6 +29,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_lookaround_regex_patterns, + is_object_schema, native_redacted_thinking_data, parse_content_for_reasoning, tool_with_sanitized_parameters, @@ -53,9 +54,16 @@ from litellm.llms.anthropic.chat.transformation import ( tools_without_unsupported_tool_search, ) from litellm.llms.anthropic.common_utils import AnthropicModelInfo -from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images -from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround, bedrock_supports_tool_search +from litellm.llms.bedrock.common_utils import ( + bedrock_chat_uses_native_structured_output, + bedrock_cleaned_output_schema, + bedrock_model_supports_regex_lookaround, + bedrock_native_output_format, + bedrock_supports_tool_search, + deployment_model_info, +) from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, bedrock_request_metadata_is_owned, @@ -939,17 +947,22 @@ class AmazonConverseConfig(BaseConfig): return _tool @staticmethod - def _supports_native_structured_outputs(model: str, custom_llm_provider: str | None = None) -> bool: - """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat). + def _supports_native_structured_outputs( + model: str, + custom_llm_provider: str | None, + thinking_requested: bool, + model_info: Mapping[str, object] | None, + ) -> bool: + """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. - Delegates to the standard ``supports_native_structured_output`` utility - which looks up the flag in ``litellm.model_cost`` via - ``_get_model_info_helper``. + ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and + ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the + request asks for thinking, which the json tool path would drop. Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html """ - from litellm.utils import supports_native_structured_output - - return supports_native_structured_output(model=model, custom_llm_provider=custom_llm_provider) + return bedrock_chat_uses_native_structured_output( + model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info + ) @staticmethod def _add_additional_properties_to_schema(schema: dict) -> dict: @@ -964,9 +977,9 @@ class AmazonConverseConfig(BaseConfig): if not isinstance(schema, dict): return schema - result: Final = dict(schema) + result: Final[dict[str, object]] = dict(schema) - if result.get("type") == "object" and "additionalProperties" not in result: + if is_object_schema(result) and "additionalProperties" not in result: result["additionalProperties"] = False # Recurse into nested schemas @@ -977,16 +990,46 @@ class AmazonConverseConfig(BaseConfig): if "items" in result and isinstance(result["items"], dict): result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"]) for defs_key in ("$defs", "definitions"): - if defs_key in result and isinstance(result[defs_key], dict): + if isinstance(definitions := result.get(defs_key), dict): result[defs_key] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in definitions.items() } for key in ("anyOf", "allOf", "oneOf"): - if key in result and isinstance(result[key], list): - result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]] + if isinstance(variants := result.get(key), list): + result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in variants] return result + def _native_output_schema( + self, + model: str, + json_schema: Mapping[str, object], + thinking_requested: bool, + model_info: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if not self._supports_native_structured_outputs( + model, self.custom_llm_provider, thinking_requested, model_info + ): + return None + if BedrockModelInfo.get_base_model(model).startswith("anthropic"): + return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse") + return json_schema + + def _output_config_for_anthropic_format( + self, model: str, output_config_format: Mapping[str, object], model_info: Mapping[str, object] | None + ) -> OutputConfigBlock | None: + native_format: Final = bedrock_native_output_format( + model, output_config_format, self.custom_llm_provider or "bedrock_converse", model_info + ) + if native_format is None: + return None + schema, name, description = (native_format.get(key) for key in ("schema", "name", "description")) + return AmazonConverseConfig._create_output_config_for_response_format( + json_schema={**schema} if isinstance(schema, Mapping) else None, + name=name if isinstance(name, str) else None, + description=description if isinstance(description, str) else None, + ) + @staticmethod def _create_output_config_for_response_format( json_schema: dict | None = None, @@ -1046,6 +1089,7 @@ class AmazonConverseConfig(BaseConfig): optional_params: dict, model: str, drop_params: bool, + model_info: Mapping[str, object] | None = None, ) -> dict: is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) base_model: Final = BedrockModelInfo.get_base_model(model) @@ -1062,6 +1106,7 @@ class AmazonConverseConfig(BaseConfig): optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled, + model_info=model_info, ) if param == "max_tokens" or param == "max_completion_tokens": optional_params["maxTokens"] = ( @@ -1273,6 +1318,7 @@ class AmazonConverseConfig(BaseConfig): optional_params: dict, non_default_params: dict, is_thinking_enabled: bool, + model_info: Mapping[str, object] | None = None, ) -> dict: """ Handles translation of response_format parameter to Bedrock format. @@ -1296,13 +1342,23 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params - if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None: + native_schema: Final = ( + self._native_output_schema( + model, + json_schema, + self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)), + model_info, + ) + if json_schema is not None + else None + ) + if native_schema is not None: # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through # to the tool-call path below. output_config: Final = self._create_output_config_for_response_format( - json_schema=json_schema, + json_schema=dict(native_schema), name=name, description=description, ) @@ -1644,7 +1700,11 @@ class AmazonConverseConfig(BaseConfig): tool_choice.pop("type", None) def _prepare_request_params( - self, optional_params: dict, model: str, drop_params: bool = False + self, + optional_params: dict, + model: str, + drop_params: bool = False, + model_info: Mapping[str, object] | None = None, ) -> tuple[dict, dict, dict, OutputConfigBlock | None]: """Prepare and separate request parameters.""" # Consume the internal ``_output_config_normalized`` marker set by @@ -1687,17 +1747,12 @@ class AmazonConverseConfig(BaseConfig): and output_config_format is not None and output_config_format.get("type") == "json_schema" and base_model.startswith("anthropic") - and self._supports_native_structured_outputs(model, self.custom_llm_provider) ): - output_config = self._create_output_config_for_response_format( - json_schema=output_config_format.get("schema"), - name=output_config_format.get("name"), - description=output_config_format.get("description"), - ) - elif output_config is None and output_config_format is not None: + output_config = self._output_config_for_anthropic_format(model, output_config_format, model_info) + if output_config is None and output_config_format is not None: litellm.verbose_logger.warning( "Bedrock Converse: dropping `output_config.format` for model=%s — " - "model does not advertise `supports_native_structured_output` in " + "model does not take native structured output for this schema per " "model_prices_and_context_window.json. The schema will not be " "enforced; pass `response_format` to use the synthetic tool-call " "fallback.", @@ -1987,7 +2042,7 @@ class AmazonConverseConfig(BaseConfig): additional_request_params, request_metadata, output_config, - ) = self._prepare_request_params(optional_params, model, drop_params) + ) = self._prepare_request_params(optional_params, model, drop_params, deployment_model_info(litellm_params)) original_tools: Final = inference_params.pop("tools", []) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index 8d1ff1d2bd9..b94435345d3 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -92,8 +92,11 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): optional_params: dict, model: str, drop_params: bool, + model_info: Mapping[str, object] | None = None, ) -> dict: - return AmazonConverseConfig.map_openai_params(self, non_default_params, optional_params, model, drop_params) + return AmazonConverseConfig.map_openai_params( + self, non_default_params, optional_params, model, drop_params, model_info + ) def transform_request( self, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index fc6088ddc66..ba0bb607975 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation ) from litellm.llms.bedrock.common_utils import ( apply_bedrock_invoke_structured_output, + bedrock_chat_uses_native_structured_output, + bedrock_cleaned_output_schema, bedrock_supports_tool_search, + deployment_model_info, get_anthropic_beta_from_headers, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, @@ -49,6 +52,7 @@ else: LiteLLMLoggingObj = Any _NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object]) +_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): @@ -71,6 +75,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_version: str = "bedrock-2023-05-31" + def __init__(self, model_info: Mapping[str, object] | None = None) -> None: + super().__init__() + self._deployment_model_info: Final = model_info + @property def custom_llm_provider(self) -> str | None: return "bedrock" @@ -81,10 +89,18 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def get_supported_openai_params(self, model: str) -> list[str]: return AnthropicConfig.get_supported_openai_params(self, model) - def uses_native_structured_output(self, model: str) -> bool: - from litellm.utils import supports_native_structured_output + def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: + return AnthropicConfig.native_output_schema(json_schema) - return supports_native_structured_output(model=model, custom_llm_provider="bedrock") + def uses_native_structured_output_for_schema( + self, + model: str, + json_schema: Mapping[str, object] | None, + request_params: Mapping[str, object], + ) -> bool: + return bedrock_chat_uses_native_structured_output( + model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info + ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) def map_openai_params( self, @@ -240,6 +256,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): apply_bedrock_invoke_structured_output( model=model, request_body=anthropic_request, + model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)), ) strip_unsupported_bedrock_invoke_output_config_keys( model=model, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 5862b05a619..17a8192ca81 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -29,6 +29,7 @@ import litellm from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import +from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -249,15 +250,99 @@ def convert_bedrock_invoke_output_format_to_inline_schema( request_body["messages"] = new_messages -def _bedrock_model_supports(model: str, key: str) -> bool: +def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "bedrock") -> bool: from litellm.utils import supports_factory - return supports_factory(model=model, custom_llm_provider="bedrock", key=key) + return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key) + + +_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object]) + + +def deployment_model_info(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]: + model_info: Final = (litellm_params or {}).get("model_info") + return _DEPLOYMENT_MODEL_INFO.validate_python(model_info) if isinstance(model_info, Mapping) else {} + + +def _bedrock_native_output_opt_in( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None +) -> bool: + deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output") + if isinstance(deployment_flag, bool): + return deployment_flag + return _bedrock_model_supports(model, "supports_bedrock_native_structured_output", custom_llm_provider) + + +def bedrock_supports_native_structured_output( + model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None +) -> bool: + return _bedrock_model_supports( + model, "supports_native_structured_output", custom_llm_provider + ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + + +def bedrock_chat_uses_native_structured_output( + model: str, + thinking_requested: bool, + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> bool: + return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or ( + thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + ) + + +BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 + + +def bedrock_cleaned_output_schema( + model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" +) -> Mapping[str, object] | None: + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + if ( + not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) + and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + ): + return None + return AnthropicConfig.native_output_schema(schema) + + +def bedrock_native_output_schema( + model: str, + schema: Mapping[str, object], + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> Mapping[str, object] | None: + native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider) + if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider): + return schema if native_schema is None else native_schema + if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info): + return native_schema + return None + + +def bedrock_native_output_format( + model: str, + output_format: Mapping[str, object], + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> Mapping[str, object] | None: + schema: Final = output_format.get("schema") + if not isinstance(schema, Mapping): + return ( + output_format if bedrock_supports_native_structured_output(model, custom_llm_provider, model_info) else None + ) + native_schema: Final = bedrock_native_output_schema(model, schema, custom_llm_provider, model_info) + if native_schema is None: + return None + return {**output_format, "schema": native_schema} def apply_bedrock_invoke_structured_output( model: str, request_body: dict[str, object], # mutable-ok: edited in place like siblings + model_info: Mapping[str, object] | None = None, ) -> None: """ Route Anthropic structured-output params to what the Bedrock model supports. @@ -277,17 +362,18 @@ def apply_bedrock_invoke_structured_output( if schema_format is None: return - if _bedrock_model_supports(model, "supports_native_structured_output"): + native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info) + if native_format is not None: existing_output_config: Final = request_body.get("output_config") if isinstance(existing_output_config, dict): - existing_output_config["format"] = schema_format + existing_output_config["format"] = native_format else: - request_body["output_config"] = {"format": schema_format} # rebind-ok: out-param + request_body["output_config"] = {"format": native_format} # rebind-ok: out-param return verbose_logger.warning( - "Bedrock Invoke: model=%s does not advertise `supports_native_structured_output` " - "in model_prices_and_context_window.json, so the JSON schema was inlined into " + "Bedrock Invoke: model=%s does not take native structured output for this schema " + "per model_prices_and_context_window.json, so the JSON schema was inlined into " "the last user message and is NOT enforced by the model.", model, ) @@ -1296,7 +1382,6 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: _BEDROCK_MODEL_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$") -_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object]) def bedrock_converse_supports_strict_tools(model: str) -> bool: @@ -1327,8 +1412,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ entry of ``model`` itself. A model nobody flagged keeps its schema as sent. """ params: Final = litellm_params or {} - model_info: Final = _DEPLOYMENT_MODEL_INFO.validate_python(params.get("model_info") or {}) - deployment_flag: Final = model_info.get("supports_regex_lookaround") + deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") if isinstance(deployment_flag, bool): return deployment_flag base_model: Final = params.get("base_model") diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 95e81fa3ee7..96a7186b136 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -3,6 +3,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -69,6 +70,9 @@ else: LiteLLMLoggingObj = Any +_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + class AmazonAnthropicClaudeMessagesConfig( AnthropicMessagesConfig, AmazonInvokeConfig, @@ -751,6 +755,7 @@ class AmazonAnthropicClaudeMessagesConfig( apply_bedrock_invoke_structured_output( model=model, request_body=anthropic_messages_request, + model_info=_MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)), ) normalize_bedrock_opus_output_config_effort( model=model, diff --git a/litellm/main.py b/litellm/main.py index 23cd3c40250..3eae64d828e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4132,6 +4132,7 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) @@ -5695,7 +5696,9 @@ def completion( and provider_config.accepts_reasoning_summary_in_reasoning_effort, ) optional_params = get_optional_params( - **{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, **non_default_params + **{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, + model_info=_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None, + **non_default_params, ) processed_non_default_params: Final = pre_process_non_default_params( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1851e818ec7..a2f3c465122 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1546,7 +1546,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1623,7 +1623,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1700,7 +1700,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1777,7 +1777,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72498,7 +72498,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72531,7 +72531,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72888,6 +72888,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72919,6 +72920,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72965,6 +72967,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73035,6 +73038,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73071,6 +73075,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73114,6 +73119,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73155,6 +73161,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73196,6 +73203,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73318,6 +73326,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73361,6 +73370,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74333,6 +74343,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74369,6 +74380,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74420,6 +74432,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74500,6 +74513,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74541,6 +74555,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74589,6 +74604,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74635,6 +74651,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74681,6 +74698,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74731,6 +74749,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74779,6 +74798,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c9925580190..8e56ebdc39b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -204,6 +204,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_pdf_input: bool | None supports_native_streaming: bool | None supports_native_structured_output: bool | None + supports_bedrock_native_structured_output: ReadOnly[bool | None] supports_parallel_function_calling: bool | None supports_web_search: bool | None supports_reasoning: bool | None @@ -4047,7 +4048,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( - {"supports_regex_lookaround", "supports_assistant_prefill"} + {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} ) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( diff --git a/litellm/utils.py b/litellm/utils.py index 04c1e628288..608aa9fd112 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4575,6 +4575,7 @@ def get_optional_params( store: bool | None = None, prompt_cache_key: str | None = None, base_model: str | None = None, + model_info: Mapping[str, object] | None = None, **kwargs: object, ): drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string @@ -4585,6 +4586,7 @@ def get_optional_params( # non_default_params / _check_valid_arg — it's a routing hint, not an # OpenAI param. passed_params.pop("base_model", None) + passed_params.pop("model_info", None) provider_config: BaseConfig | None = None if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( @@ -4824,6 +4826,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, drop_params=bool(drop_params), + model_info=model_info, ) elif bedrock_route == "openai": optional_params = litellm.AmazonBedrockOpenAIConfig().map_openai_params( @@ -4841,7 +4844,7 @@ def get_optional_params( drop_params=bool(drop_params), ) else: - optional_params = litellm.AmazonAnthropicClaudeConfig().map_openai_params( + optional_params = litellm.AmazonAnthropicClaudeConfig(model_info=model_info).map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, @@ -6465,6 +6468,9 @@ def get_model_info_helper( supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), supports_native_streaming=_model_info.get("supports_native_streaming", None), supports_native_structured_output=_model_info.get("supports_native_structured_output", None), + supports_bedrock_native_structured_output=_model_info.get( + "supports_bedrock_native_structured_output", None + ), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1851e818ec7..a2f3c465122 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1546,7 +1546,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1623,7 +1623,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1700,7 +1700,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1777,7 +1777,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72498,7 +72498,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72531,7 +72531,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72888,6 +72888,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72919,6 +72920,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -72965,6 +72967,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73035,6 +73038,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73071,6 +73075,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73114,6 +73119,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73155,6 +73161,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73196,6 +73203,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -73318,6 +73326,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -73361,6 +73370,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74333,6 +74343,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74369,6 +74380,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74420,6 +74432,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74500,6 +74513,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74541,6 +74555,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74589,6 +74604,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74635,6 +74651,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74681,6 +74698,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -74731,6 +74749,7 @@ "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -74779,6 +74798,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 4404aa8e35c..33a784e2dae 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1027,6 +1027,9 @@ "supports_bedrock_converse_tool_result_images": { "type": "boolean" }, + "supports_bedrock_native_structured_output": { + "type": "boolean" + }, "supports_bedrock_runtime_chat_completions_inline_reasoning": { "type": "boolean" }, diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py new file mode 100644 index 00000000000..f389e53d132 --- /dev/null +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -0,0 +1,820 @@ +import json +import re +import uuid +from collections.abc import Iterable, Iterator, Mapping +from itertools import chain +from typing import Final +from urllib.parse import unquote + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, list_value, object_value, string_value +from integration._support.upstream import aws_event_stream_frame +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_HAIKU: Final = "us.anthropic.claude-haiku-5-5" +_FABLE: Final = "us.anthropic.claude-fable-5" +_SONNET: Final = "us.anthropic.claude-sonnet-4-6" +_OPUS: Final = "us.anthropic.claude-opus-5-5" +_OPT_IN: Final = "au.anthropic.claude-opus-4-8" +_DEEPSEEK: Final = "deepseek.v3.2" +_AWS: Final[dict[str, JsonValue]] = { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", +} +_NO_CACHE: Final[dict[str, JsonValue]] = {"cache": {"no-cache": True}, "num_retries": 0} +_JSON_TOOL: Final = "json_tool_call" +_USER_TOOL: Final = "lookup" +_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" +_ADAPTIVE: Final[dict[str, JsonValue]] = {"type": "adaptive"} +_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) +_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" +_USAGE: Final[dict[str, JsonValue]] = {"inputTokens": 11, "outputTokens": 7, "totalTokens": 18} +_MARKER: Final = re.compile(r"probe ([0-9a-f]{32})") +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_FABLE_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) +_UNSUPPORTED_KEYWORDS: Final = ( + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "maxItems", + "uniqueItems", + "minProperties", + "maxProperties", + "patternProperties", + "oneOf", +) +_SCHEMA_MAPS: Final = ("properties", "$defs", "definitions") +_SCHEMA_LISTS: Final = ("anyOf", "allOf", "oneOf") +_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"tier": {"type": "integer"}, "summary": {"type": "string"}}, + "required": ["tier", "summary"], + "additionalProperties": False, +} +_LIMITS_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "tier": {"type": "integer", "minimum": 1, "maximum": 5}, + "summary": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "owner": {"anyOf": [{"$ref": "#/$defs/Person"}, {"type": "null"}]}, + "backup": {"type": ["object", "null"], "properties": {"name": {"type": "string"}}, "required": ["name"]}, + }, + "required": ["tier", "summary", "tags", "owner", "backup"], + "$defs": {"Person": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}, +} +_OPEN_COUNTS: Final[dict[str, JsonValue]] = {"type": "object", "additionalProperties": {"type": "integer"}} +_DICT_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"store": {"type": "string"}, "counts": _OPEN_COUNTS}, + "required": ["store", "counts"], + "additionalProperties": False, +} +_NULLABLE_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "summary": {"type": "string"}, + "backup": {"type": ["object", "null"], "properties": {"name": {"type": "string"}}, "required": ["name"]}, + }, + "required": ["summary", "backup"], + "additionalProperties": False, +} +_CLOSED_NULLABLE_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "summary": {"type": "string"}, + "backup": { + "type": ["object", "null"], + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "additionalProperties": False, + }, + }, + "required": ["summary", "backup"], + "additionalProperties": False, +} +_UNCLEANED_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"tier": {"type": "integer", "minimum": 1, "maximum": 5}, "summary": {"type": "string"}}, + "required": ["tier", "summary"], + "additionalProperties": False, +} +_NUMERIC_FORMATS_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "tier": {"type": "integer", "format": "int32"}, + "score": {"type": "number", "format": "double"}, + "summary": {"type": "string"}, + }, + "required": ["tier", "score", "summary"], + "additionalProperties": False, +} +_WIDE_FIELDS: Final = tuple(f"field_{index:02d}" for index in range(65)) +_WIDE_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {name: {"type": "string"} for name in _WIDE_FIELDS}, + "required": list(_WIDE_FIELDS), + "additionalProperties": False, +} +_USER_TOOL_SPEC: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": _USER_TOOL, + "description": "Look up a record", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}, + }, +} + + +def _payload(marker_hex: str) -> dict[str, JsonValue]: + return {"tier": 2, "summary": marker_hex} + + +def _marker() -> tuple[str, str]: + marker_hex: Final = uuid.uuid4().hex + return f"probe {marker_hex}", marker_hex + + +def _response_format(schema: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {"type": "json_schema", "json_schema": {"name": "risk", "schema": dict(schema)}} + + +def _model_id(request: Request) -> str: + return unquote(request.target).split("/model/", 1)[1].rsplit("/", 1)[0] + + +def _is_invoke(request: Request) -> bool: + return unquote(request.target).rsplit("/", 1)[1].startswith("invoke") + + +def _is_stream(request: Request) -> bool: + return unquote(request.target).endswith(("/converse-stream", "/invoke-with-response-stream")) + + +def _thinking_of(body: Mapping[str, JsonValue], invoke: bool) -> JsonValue: + if invoke: + return body.get("thinking") + return object_value(body.get("additionalModelRequestFields") or {}).get("thinking") + + +def _thinking_requested(body: Mapping[str, JsonValue], invoke: bool) -> bool: + thinking: Final = _thinking_of(body, invoke) + return isinstance(thinking, dict) and thinking.get("type") in _THINKING_TYPES + + +def _native_schema(body: Mapping[str, JsonValue], invoke: bool) -> dict[str, JsonValue] | None: + if invoke: + output_config: Final = body.get("output_config") + if not isinstance(output_config, dict) or "format" not in output_config: + return None + return object_value(object_value(object_value(output_config)["format"])["schema"]) + text_format: Final = object_value(body.get("outputConfig") or {}).get("textFormat") + if text_format is None: + return None + json_schema: Final = object_value(object_value(object_value(text_format)["structure"])["jsonSchema"]) + return _JSON.validate_json(string_value(json_schema["schema"])) + + +def _tools(body: Mapping[str, JsonValue], invoke: bool) -> dict[str, dict[str, JsonValue]]: + if invoke: + return { + string_value(object_value(tool)["name"]): object_value(object_value(tool)["input_schema"]) + for tool in list_value(body.get("tools", [])) + } + specs: Final = tuple( + object_value(object_value(tool)["toolSpec"]) + for tool in list_value(object_value(body.get("toolConfig") or {"tools": []})["tools"]) + ) + return {string_value(spec["name"]): object_value(object_value(spec["inputSchema"])["json"]) for spec in specs} + + +def _is_object_type(node: Mapping[str, JsonValue]) -> bool: + node_type: Final = node.get("type") + return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) + + +def _child_schemas(node: Mapping[str, JsonValue]) -> Iterable[JsonValue]: + mapped: Final = (node.get(key) for key in _SCHEMA_MAPS) + listed: Final = (node.get(key) for key in _SCHEMA_LISTS) + single: Final = (node.get(key) for key in ("items", "additionalProperties")) + return chain( + chain.from_iterable(children.values() for children in mapped if isinstance(children, dict)), + chain.from_iterable(variants for variants in listed if isinstance(variants, list)), + (child for child in single if isinstance(child, dict)), + ) + + +def _schema_nodes(schema: JsonValue) -> Iterator[dict[str, JsonValue]]: + if not isinstance(schema, dict): + return + yield schema + yield from chain.from_iterable(_schema_nodes(child) for child in _child_schemas(schema)) + + +def _node_fault(node: Mapping[str, JsonValue]) -> str | None: + node_type: Final = node.get("type") + unsupported: Final = tuple(key for key in _UNSUPPORTED_KEYWORDS if key in node) + if unsupported: + return f"For '{node_type}' type, property '{unsupported[0]}' is not supported" + min_items: Final = node.get("minItems") + if isinstance(min_items, int) and min_items > 1: + return "For 'array' type, property 'minItems' is not supported" + if _is_object_type(node) and node.get("additionalProperties") is not False: + return "For 'object' type, 'additionalProperties' must be explicitly set to false" + return None + + +def _bedrock_schema_fault(schema: JsonValue) -> str | None: + faults: Final = tuple(fault for node in _schema_nodes(schema) if (fault := _node_fault(node)) is not None) + if not faults: + return None + return f"The model returned the following errors: output_config.format.schema: {faults[0]}" + + +def _rejection(model_id: str, native: dict[str, JsonValue] | None) -> str | None: + if native is None: + return None + if model_id == _FABLE: + return _FABLE_REJECTION + if "anthropic." not in model_id: + return None + return _bedrock_schema_fault(native) + + +def _reasoning_block(marker_hex: str, invoke: bool) -> dict[str, JsonValue]: + if invoke: + return {"type": "thinking", "thinking": f"reasoning {marker_hex}", "signature": _SIGNATURE} + return {"reasoningContent": {"reasoningText": {"text": f"reasoning {marker_hex}", "signature": _SIGNATURE}}} + + +def _answer_block(name: str | None, marker_hex: str, invoke: bool) -> dict[str, JsonValue]: + tool_input: Final[dict[str, JsonValue]] = {"q": marker_hex} if name == _USER_TOOL else _payload(marker_hex) + if name is None and invoke: + return {"type": "text", "text": json.dumps(_payload(marker_hex))} + if name is None: + return {"text": json.dumps(_payload(marker_hex))} + if invoke: + return {"type": "tool_use", "id": f"toolu_{marker_hex}", "name": name, "input": tool_input} + return {"toolUse": {"toolUseId": f"tooluse_{marker_hex}", "name": name, "input": tool_input}} + + +def _converse_frames(blocks: tuple[dict[str, JsonValue], ...], stop_reason: str) -> bytes: + def frame(event_type: str, payload: Mapping[str, JsonValue]) -> bytes: + return aws_event_stream_frame( + {":event-type": event_type, ":content-type": "application/json", ":message-type": "event"}, + json.dumps(payload).encode(), + ) + + def block_frames(index: int, block: dict[str, JsonValue]) -> bytes: + if "reasoningContent" in block: + reasoning: Final = object_value(object_value(block["reasoningContent"])["reasoningText"]) + return b"".join( + ( + frame( + "contentBlockDelta", + {"delta": {"reasoningContent": {"text": reasoning["text"]}}, "contentBlockIndex": index}, + ), + frame( + "contentBlockDelta", + { + "delta": {"reasoningContent": {"signature": reasoning["signature"]}}, + "contentBlockIndex": index, + }, + ), + frame("contentBlockStop", {"contentBlockIndex": index}), + ) + ) + if "toolUse" in block: + tool_use: Final = object_value(block["toolUse"]) + return b"".join( + ( + frame( + "contentBlockStart", + { + "start": {"toolUse": {"toolUseId": tool_use["toolUseId"], "name": tool_use["name"]}}, + "contentBlockIndex": index, + }, + ), + frame( + "contentBlockDelta", + {"delta": {"toolUse": {"input": json.dumps(tool_use["input"])}}, "contentBlockIndex": index}, + ), + frame("contentBlockStop", {"contentBlockIndex": index}), + ) + ) + return b"".join( + ( + frame("contentBlockDelta", {"delta": {"text": block["text"]}, "contentBlockIndex": index}), + frame("contentBlockStop", {"contentBlockIndex": index}), + ) + ) + + return b"".join( + ( + frame("messageStart", {"role": "assistant"}), + *(block_frames(index, block) for index, block in enumerate(blocks)), + frame("messageStop", {"stopReason": stop_reason}), + frame("metadata", {"usage": _USAGE, "metrics": {"latencyMs": 1}}), + ) + ) + + +def _called_tool(tools: Iterable[str], native: bool) -> str | None: + names: Final = frozenset(tools) + if _USER_TOOL in names: + return _USER_TOOL + if native or _JSON_TOOL not in names: + return None + return _JSON_TOOL + + +def _error_reply(message: str) -> Reply: + return Reply(status=400, body=json.dumps({"message": message}).encode()) + + +def _bedrock(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + invoke: Final = _is_invoke(request) + found: Final = _MARKER.search(json.dumps(body)) + assert found is not None, body + marker_hex: Final = found.group(1) + native: Final = _native_schema(body, invoke) + rejection: Final = _rejection(_model_id(request), native) + if rejection is not None: + return _error_reply(rejection) + called: Final = _called_tool(_tools(body, invoke).keys(), native is not None) + reasoning: Final = (_reasoning_block(marker_hex, invoke),) if _thinking_requested(body, invoke) else () + blocks: Final = (*reasoning, _answer_block(called, marker_hex, invoke)) + stop_reason: Final = "tool_use" if called is not None else "end_turn" + if invoke: + message: Final[dict[str, JsonValue]] = { + "id": f"msg_{marker_hex}", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": list(blocks), + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + return Reply(body=json.dumps(message).encode()) + if _is_stream(request): + return Reply(body=_converse_frames(blocks, stop_reason), content_type=_EVENT_STREAM) + converse: Final[dict[str, JsonValue]] = { + "output": {"message": {"role": "assistant", "content": list(blocks)}}, + "stopReason": stop_reason, + "usage": _USAGE, + "metrics": {"latencyMs": 1}, + } + return Reply(body=json.dumps(converse).encode()) + + +def _deployment( + scenario: Scenario, wire: Wire, route: str, backend: str, model_info: Mapping[str, JsonValue] | None = None +) -> str: + return scenario.model(model=f"bedrock/{route}/{backend}", api_base=wire.url, model_info=model_info, **_AWS) + + +def _chat_body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": marker}], "max_tokens": 2048, **_NO_CACHE, **extra} + + +def _messages_body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 2048, "messages": [{"role": "user", "content": marker}], **_NO_CACHE, **extra} + + +def _one_call(wire: Wire, marker_hex: str) -> tuple[Request, dict[str, JsonValue]]: + (request,) = wire.drain() + body: Final = _JSON.validate_json(request.body) + assert marker_hex in json.dumps(body), body + return request, body + + +def _chat_message(response: httpx.Response) -> dict[str, JsonValue]: + assert response.status_code == 200, response.text + choices: Final = list_value(_JSON.validate_json(response.content)["choices"]) + return object_value(object_value(choices[0])["message"]) + + +def _stream_message(response: httpx.Response) -> dict[str, JsonValue]: + assert response.status_code == 200, response.text + deltas: Final = tuple( + object_value(object_value(list_value(_JSON.validate_json(line[6:])["choices"])[0])["delta"]) + for line in response.text.splitlines() + if line.startswith("data: {") + ) + return { + "content": "".join(string_value(delta["content"]) for delta in deltas if delta.get("content")), + "reasoning_content": "".join( + string_value(delta["reasoning_content"]) for delta in deltas if delta.get("reasoning_content") + ), + } + + +def _content_json(message: Mapping[str, JsonValue]) -> JsonValue: + return _JSON_VALUE.validate_json(string_value(message["content"])) + + +def _responses_items(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]: + assert response.status_code == 200, response.text + return tuple(object_value(item) for item in list_value(_JSON.validate_json(response.content)["output"])) + + +def _responses_text(items: tuple[dict[str, JsonValue], ...]) -> JsonValue: + (message,) = tuple(item for item in items if item.get("type") == "message") + (text,) = tuple( + object_value(part) for part in list_value(message["content"]) if object_value(part).get("type") == "output_text" + ) + return _JSON_VALUE.validate_json(string_value(text["text"])) + + +def _messages_blocks(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]: + assert response.status_code == 200, response.text + return tuple(object_value(block) for block in list_value(_JSON.validate_json(response.content)["content"])) + + +def _messages_text(blocks: tuple[dict[str, JsonValue], ...]) -> JsonValue: + (text,) = tuple(block for block in blocks if block.get("type") == "text") + return _JSON_VALUE.validate_json(string_value(text["text"])) + + +def _refs(schema: JsonValue) -> frozenset[str]: + return frozenset(string_value(node["$ref"]) for node in _schema_nodes(schema) if "$ref" in node) + + +def _resolves(schema: Mapping[str, JsonValue], ref: str) -> bool: + prefix: Final = "#/$defs/" + return ref.startswith(prefix) and ref.removeprefix(prefix) in object_value(schema.get("$defs") or {}) + + +def _assert_native_without_tools(body: Mapping[str, JsonValue], invoke: bool, schema: Mapping[str, JsonValue]) -> None: + assert _native_schema(body, invoke) == schema, body + assert _JSON_TOOL not in _tools(body, invoke), body + + +@pytest.mark.parametrize( + ("route", "stream"), + [("converse", False), ("converse", True), ("invoke", False)], + ids=["converse", "converse_stream", "invoke"], +) +def test_thinking_request_on_a_bedrock_native_model_gets_native_output_and_keeps_reasoning( + gateway: Gateway, route: str, stream: bool +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _HAIKU) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, marker, stream=stream, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)), + ) + message: Final = _stream_message(response) if stream else _chat_message(response) + assert _content_json(message) == _payload(marker_hex), response.text + assert message["reasoning_content"] == f"reasoning {marker_hex}", response.text + request, body = _one_call(wire, marker_hex) + invoke: Final = route == "invoke" + assert _is_stream(request) is stream, request.target + _assert_native_without_tools(body, invoke, _SCHEMA) + assert _thinking_requested(body, invoke), body + + +def test_responses_reasoning_effort_on_a_bedrock_native_model_gets_native_output_and_keeps_reasoning( + gateway: Gateway, +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _HAIKU) + response: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "input": marker, + "max_output_tokens": 2048, + "reasoning": {"effort": "low"}, + "text": {"format": {"type": "json_schema", "name": "risk", "schema": _SCHEMA}}, + **_NO_CACHE, + }, + ) + items: Final = _responses_items(response) + assert _responses_text(items) == _payload(marker_hex), response.text + (reasoning,) = tuple(item for item in items if item.get("type") == "reasoning") + assert f"reasoning {marker_hex}" in json.dumps(reasoning), response.text + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, False, _SCHEMA) + assert _thinking_requested(body, False), body + + +def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output_and_keeps_reasoning( + gateway: Gateway, +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _HAIKU) + response: Final = gateway.request( + "POST", + "/v1/messages", + _messages_body(model, marker, thinking=_ADAPTIVE, output_format={"type": "json_schema", "schema": _SCHEMA}), + ) + blocks: Final = _messages_blocks(response) + assert _messages_text(blocks) == _payload(marker_hex), response.text + (thinking,) = tuple(block for block in blocks if block.get("type") == "thinking") + assert thinking["thinking"] == f"reasoning {marker_hex}", response.text + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, False, _SCHEMA) + assert _thinking_requested(body, False), body + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gateway: Gateway, route: str) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _HAIKU) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) + ) + message: Final = _chat_message(response) + assert _content_json(message) == _payload(marker_hex), response.text + assert not message.get("tool_calls"), response.text + _, body = _one_call(wire, marker_hex) + invoke: Final = route == "invoke" + assert _native_schema(body, invoke) is None, body + assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body + + +@pytest.mark.parametrize( + ("backend", "model_info"), + [(_OPUS, None), (_OPT_IN, {"supports_bedrock_native_structured_output": True})], + ids=["neither_key", "deployment_opt_in"], +) +def test_thinking_request_follows_the_bedrock_native_key_the_model_resolves_to( + gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None +) -> None: + marker, marker_hex = _marker() + native: Final = model_info is not None + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", backend, model_info) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)), + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + assert (_native_schema(body, False) == _SCHEMA) is native, body + assert (_JSON_TOOL in _tools(body, False)) is not native, body + + +def _opt_in_answer(gateway: Gateway, entry: str, model: str, marker: str) -> JsonValue: + output_format: Final[dict[str, JsonValue]] = {"type": "json_schema", "schema": _SCHEMA} + match entry: + case "converse_chat" | "invoke_chat": + chat: Final = _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)) + return _content_json(_chat_message(gateway.request("POST", "/v1/chat/completions", chat))) + case "converse_responses": + responses: Final[dict[str, JsonValue]] = { + "model": model, + "input": marker, + "max_output_tokens": 2048, + "reasoning": {"effort": "low"}, + "text": {"format": {**output_format, "name": "risk"}}, + **_NO_CACHE, + } + return _responses_text(_responses_items(gateway.request("POST", "/v1/responses", responses))) + case "converse_messages": + adapted: Final = _messages_body(model, marker, thinking=_ADAPTIVE, output_format=output_format) + return _messages_text(_messages_blocks(gateway.request("POST", "/v1/messages", adapted))) + case _: + invoked: Final = _messages_body(model, marker, output_config={"format": output_format}) + return _messages_text(_messages_blocks(gateway.request("POST", "/v1/messages", invoked))) + + +def _sent_native_schema(gateway: Gateway, wire: Wire, entry: str, model: str) -> dict[str, JsonValue] | None: + marker, marker_hex = _marker() + assert _opt_in_answer(gateway, entry, model, marker) == _payload(marker_hex) + _, body = _one_call(wire, marker_hex) + invoke: Final = entry.startswith("invoke") + native: Final = _native_schema(body, invoke) + assert native is None or _JSON_TOOL not in _tools(body, invoke), body + return native + + +@pytest.mark.parametrize( + "entry", ["converse_chat", "converse_responses", "converse_messages", "invoke_chat", "invoke_messages"] +) +def test_a_deployment_native_output_opt_in_stays_on_the_deployment_that_sets_it(gateway: Gateway, entry: str) -> None: + route: Final = entry.split("_", 1)[0] + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + opted_in: Final = _deployment( + scenario, wire, route, _OPT_IN, {"supports_bedrock_native_structured_output": True} + ) + sibling: Final = _deployment(scenario, wire, route, _OPT_IN) + assert _sent_native_schema(gateway, wire, entry, sibling) is None + assert _sent_native_schema(gateway, wire, entry, opted_in) == _SCHEMA + + +def test_fable_5_json_schema_takes_the_json_tool_instead_of_the_native_output_bedrock_rejects( + gateway: Gateway, +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _FABLE) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + assert "outputConfig" not in body, body + assert _tools(body, False)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body + + +def _limits_answer(gateway: Gateway, entry: str, model: str, marker: str) -> JsonValue: + output_config: Final[dict[str, JsonValue]] = {"format": {"type": "json_schema", "schema": _LIMITS_SCHEMA}} + match entry: + case "converse_chat" | "invoke_chat": + body: Final = _chat_body(model, marker, response_format=_response_format(_LIMITS_SCHEMA)) + return _content_json(_chat_message(gateway.request("POST", "/v1/chat/completions", body))) + case "converse_output_config": + chat: Final = _chat_body(model, marker, output_config=output_config) + return _content_json(_chat_message(gateway.request("POST", "/v1/chat/completions", chat))) + case _: + messages: Final = _messages_body(model, marker, output_config=output_config) + return _messages_text(_messages_blocks(gateway.request("POST", "/v1/messages", messages))) + + +@pytest.mark.parametrize("entry", ["converse_chat", "converse_output_config", "invoke_chat", "invoke_messages"]) +def test_native_schema_reaches_bedrock_with_limits_in_descriptions_and_every_object_closed( + gateway: Gateway, entry: str +) -> None: + marker, marker_hex = _marker() + route: Final = entry.split("_", 1)[0] + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _SONNET) + assert _limits_answer(gateway, entry, model, marker) == _payload(marker_hex) + _, body = _one_call(wire, marker_hex) + schema: Final = _native_schema(body, route == "invoke") + assert schema is not None, body + assert _bedrock_schema_fault(schema) is None, schema + properties: Final = object_value(schema["properties"]) + assert object_value(properties["tags"])["minItems"] == 1, schema + assert {"1", "5"} <= set(re.findall(r"\d+", string_value(object_value(properties["tier"])["description"]))) + assert all(_resolves(schema, ref) for ref in _refs(schema)), schema + assert _JSON_TOOL not in _tools(body, route == "invoke"), body + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_open_object_schema_takes_the_json_tool_on_a_native_model_and_keeps_the_dict_open( + gateway: Gateway, route: str +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _SONNET) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_DICT_SCHEMA)) + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + invoke: Final = route == "invoke" + assert _native_schema(body, invoke) is None, body + tool_schema: Final = _tools(body, invoke)[_JSON_TOOL] + assert object_value(tool_schema["properties"])["counts"] == _OPEN_COUNTS, tool_schema + + +def test_thinking_with_a_caller_tool_on_a_bedrock_native_model_sends_only_the_callers_tool(gateway: Gateway) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _HAIKU) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body( + model, + marker, + thinking=_ADAPTIVE, + tools=[_USER_TOOL_SPEC], + response_format=_response_format(_SCHEMA), + ), + ) + (call,) = list_value(_chat_message(response)["tool_calls"]) + function: Final = object_value(object_value(call)["function"]) + assert function["name"] == _USER_TOOL, response.text + assert json.loads(string_value(function["arguments"])) == {"q": marker_hex}, response.text + _, body = _one_call(wire, marker_hex) + assert _tools(body, False).keys() == {_USER_TOOL}, body + assert _native_schema(body, False) == _SCHEMA, body + assert _thinking_requested(body, False), body + + +@pytest.mark.parametrize( + ("schema", "expected"), + [(_UNCLEANED_SCHEMA, _UNCLEANED_SCHEMA), (_NULLABLE_SCHEMA, _CLOSED_NULLABLE_SCHEMA)], + ids=["limits_kept", "nullable_object_closed"], +) +def test_non_anthropic_native_model_gets_the_callers_schema_with_objects_closed( + gateway: Gateway, schema: dict[str, JsonValue], expected: dict[str, JsonValue] +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _DEEPSEEK) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(schema)) + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + assert _native_schema(body, False) == expected, body + + +@pytest.mark.parametrize( + ("schema", "native"), + [ + (_SCHEMA, True), + (_DICT_SCHEMA, False), + ], + ids=["closed_schema_native", "open_object_inlined"], +) +def test_messages_on_bedrock_invoke_follows_the_bedrock_native_key_without_thinking( + gateway: Gateway, schema: dict[str, JsonValue], native: bool +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "invoke", _HAIKU) + response: Final = gateway.request( + "POST", + "/v1/messages", + _messages_body(model, marker, output_config={"format": {"type": "json_schema", "schema": schema}}), + ) + assert _messages_text(_messages_blocks(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + sent: Final = _native_schema(body, True) + assert (sent is not None) is native, body + prompt: Final = json.dumps(body["messages"]) + assert all((name in prompt) is not native for name in object_value(schema["properties"])), body + + +@pytest.mark.parametrize( + "schema", + [ + {"type": "object", "properties": ["tier", "summary"]}, + {"type": "object", "properties": {"tier": {"$ref": 5}}, "additionalProperties": False}, + {"type": "object", "$defs": [1, 2], "properties": {"tier": {"$ref": "#/$defs/Tier"}}}, + {"type": ["object", "null"]}, + {"type": "object", "properties": {"tier": {"type": "integer"}}, "required": "tier"}, + ], + ids=["properties_list", "ref_not_string", "defs_list", "type_list_without_properties", "required_string"], +) +def test_malformed_schema_on_a_native_model_reaches_bedrock_once_without_a_proxy_error( + gateway: Gateway, schema: dict[str, JsonValue] +) -> None: + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _SONNET) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(schema)) + ) + assert response.status_code < 500, response.text + _, body = _one_call(wire, marker_hex) + native: Final = _native_schema(body, False) + fault: Final = None if native is None else _bedrock_schema_fault(native) + assert response.status_code == (200 if fault is None else 400), response.text + assert fault is None or fault.split(": ", 1)[1] in response.text, response.text + assert (native is None) is (_JSON_TOOL in _tools(body, False)), body + assert gateway.request("GET", "/health/liveliness").status_code == 200 + + +@pytest.mark.parametrize( + ("route", "backend", "thinking", "schema", "native"), + [ + ("converse", _SONNET, False, _NUMERIC_FORMATS_SCHEMA, True), + ("invoke", _HAIKU, True, _NUMERIC_FORMATS_SCHEMA, True), + ("converse", _SONNET, False, _WIDE_SCHEMA, True), + ("converse", _HAIKU, True, _WIDE_SCHEMA, False), + ], + ids=["numeric_formats_old_key", "numeric_formats_new_key_invoke", "wide_old_key", "wide_new_key"], +) +def test_numeric_formats_and_wide_schemas_follow_the_bedrock_native_key( + gateway: Gateway, route: str, backend: str, thinking: bool, schema: dict[str, JsonValue], native: bool +) -> None: + marker, marker_hex = _marker() + extra: Final[dict[str, JsonValue]] = {"thinking": _ADAPTIVE} if thinking else {} + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, backend) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, marker, response_format=_response_format(schema), **extra), + ) + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + _, body = _one_call(wire, marker_hex) + invoke: Final = route == "invoke" + assert (_native_schema(body, invoke) == schema) is native, body + tools: Final = _tools(body, invoke) + assert (_JSON_TOOL in tools) is not native, body + assert native or tools[_JSON_TOOL]["properties"] == schema["properties"], body diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 1809b8ce5f8..b2aba7fb100 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -37,8 +37,11 @@ _STRUCTURED_OUTPUTS_BETA: Final = "structured-outputs-2025-11-13" _TOOL_MODEL: Final = "claude-sonnet-5" _EFFORT_GATED_MODEL: Final = "claude-haiku-4-5" _NATIVE_MODEL: Final = "claude-opus-5-5" +_FORCED_TOOL_NATIVE_MODEL: Final = "claude-haiku-5-5" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" +_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" +_THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" _LOCATION: Final = "us-east5" _VERTEX_MODELS: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}/publishers/anthropic/models/" @@ -85,6 +88,10 @@ _VALUES_SCHEMA: Final[dict[str, JsonValue]] = { "properties": {"values": {"type": "object"}}, "required": ["values"], } +_VALUES_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "valued", "schema": _VALUES_SCHEMA}, +} _ADDITIONAL_OBJECT_SCHEMA: Final[dict[str, JsonValue]] = { **_OPTIONAL_SCHEMA, "additionalProperties": {"type": "object"}, @@ -1429,6 +1436,154 @@ def test_vertex_responses_native_folds_output_format(gateway: Gateway) -> None: assert _folded_format(_one_delivery(wire, marker_hex)) == _OUTPUT_FORMAT +def _native_deployment(gateway: Gateway, scenario: Scenario, api_base: str, provider: str, backend: str) -> str: + match provider: + case "vertex_ai": + return _vertex_deployment(gateway, scenario, api_base, backend) + case "anthropic": + return scenario.model(model=f"anthropic/{backend}", api_base=api_base, api_key=_ANTHROPIC_KEY) + case "azure_ai": + return scenario.model(model=f"azure_ai/{backend}", api_base=api_base, api_key=_AZURE_KEY) + case _: + raise AssertionError(provider) + + +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"]) +def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_keeps_the_object_open( + gateway: Gateway, provider: str +) -> None: + marker, marker_hex = _marker("values") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, provider, _FORCED_TOOL_NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT) + ) + assert _chat_content(response) == _tool_input("values", marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert not _structured(sent.body), sent.body + assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + + +@pytest.mark.parametrize( + ("provider", "backend", "extra"), + [ + ("anthropic", _NATIVE_MODEL, {}), + ("vertex_ai", _NATIVE_MODEL, {}), + ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"thinking": {"type": "adaptive"}}), + ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"tools": [_USER_TOOL_SPEC]}), + ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"reasoning_effort": "none"}), + ("azure_ai", _FORCED_TOOL_NATIVE_MODEL, {"reasoning_effort": "none"}), + ], + ids=[ + "forced_tool_refused-anthropic", + "forced_tool_refused-vertex_ai", + "thinking", + "caller_tools", + "effort_none-anthropic", + "effort_none-azure_ai", + ], +) +def test_open_object_schema_keeps_native_output_when_the_json_tool_cannot_be_forced( + gateway: Gateway, provider: str, backend: str, extra: dict[str, JsonValue] +) -> None: + marker, marker_hex = _marker("values") + expected_tools: Final[frozenset[str]] = frozenset({_USER_TOOL}) if "tools" in extra else frozenset() + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, provider, backend) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {**_chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT), **extra}, + ) + assert _chat_content(response) == _payload(marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert _structured(sent.body), sent.body + assert _tool_names(sent.body) == expected_tools, sent.body + assert "tool_choice" not in sent.body, sent.body + assert ("thinking" in sent.body) is ("thinking" in extra), sent.body + + +def _thinking_then_json_tool(request: Request) -> Reply: + body: Final = _JSON_OBJECT.validate_json(request.body) + _, marker_hex = _marker_of(body) + assert _JSON_TOOL in _tool_names(body), body + thinking: Final[dict[str, JsonValue]] = { + "type": "thinking", + "thinking": f"thinking {marker_hex}", + "signature": _THINKING_SIGNATURE, + } + answer: Final = _tool_use(f"toolu_{marker_hex}", _JSON_TOOL, _payload(marker_hex)) + return Reply(body=json.dumps(_message(f"msg_{marker_hex}", [thinking, answer], "tool_use")).encode()) + + +def _json_tool_deployment(gateway: Gateway, scenario: Scenario, api_base: str, provider: str) -> str: + match provider: + case "vertex_ai": + return _vertex_deployment(gateway, scenario, api_base, _TOOL_MODEL) + case "bedrock_invoke": + return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_JSON_TOOL_MODEL) + case _: + raise AssertionError(provider) + + +@pytest.mark.parametrize("provider", ["vertex_ai", "bedrock_invoke"]) +def test_json_mode_answer_keeps_the_thinking_that_preceded_the_json_tool_call(gateway: Gateway, provider: str) -> None: + marker, marker_hex = _marker("plain") + with wire_server(_thinking_then_json_tool) as wire, gateway.scenario() as scenario: + model: Final = _json_tool_deployment(gateway, scenario, wire.url, provider) + response: Final = gateway.request( + "POST", "/v1/chat/completions", {**_chat_body(model, marker), "reasoning_effort": "high"} + ) + message: Final = _chat_message(response) + assert json.loads(string_value(message["content"])) == _payload(marker_hex), response.text + assert message.get("reasoning_content") == f"thinking {marker_hex}", response.text + (block,) = tuple(object_value(item) for item in list_value(message.get("thinking_blocks") or [])) + assert (block["thinking"], block["signature"]) == (f"thinking {marker_hex}", _THINKING_SIGNATURE), block + sent: Final = _one_delivery(wire, marker_hex) + assert "thinking" in sent.body, sent.body + assert sent.body.get("tool_choice") != _FORCED_JSON_TOOL, sent.body + + +def test_vertex_chat_native_schema_keeps_min_items_one_and_closes_a_nullable_object(gateway: Gateway) -> None: + marker, marker_hex = _marker("native") + schema: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "summary": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "backup": {"type": ["object", "null"], "properties": {"name": {"type": "string"}}, "required": ["name"]}, + }, + "required": ["summary", "tags", "backup"], + "additionalProperties": False, + } + expected: Final[dict[str, JsonValue]] = { + **schema, + "properties": { + "summary": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "backup": { + "type": ["object", "null"], + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "additionalProperties": False, + }, + }, + } + response_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "tagged", "schema": schema}, + } + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url, _NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=response_format) + ) + assert _chat_content(response) == _payload(marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert _folded_format(sent) == {"type": "json_schema", "schema": expected}, sent.body + assert "tools" not in sent.body, sent.body + + @pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) def test_vertex_messages_folds_output_format_into_output_config(gateway: Gateway, stream: bool) -> None: marker, marker_hex = _marker("native") diff --git a/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py b/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py new file mode 100644 index 00000000000..1c419e5b11c --- /dev/null +++ b/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py @@ -0,0 +1,139 @@ +import json +import re +import uuid +from collections.abc import Callable, Iterator, Mapping +from typing import Final + +import pytest +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm import ModelResponse, Router +from litellm.utils import _invalidate_model_cost_lowercase_map # pyright: ignore[reportPrivateUsage] # required cache invalidation + +_OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" +_JSON_TOOL: Final = "json_tool_call" +_MARKER: Final = re.compile(r"probe ([0-9a-f]{32})") +_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, +} +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) +_OPTIONS: Final[dict[str, JsonValue]] = { + "max_tokens": 2048, + "thinking": {"type": "adaptive"}, + "response_format": {"type": "json_schema", "json_schema": {"name": "city", "schema": _SCHEMA}}, + "num_retries": 0, +} + + +def _native(body: Mapping[str, JsonValue], invoke: bool) -> bool: + output_config: Final = body.get("output_config" if invoke else "outputConfig") + return isinstance(output_config, dict) and ("format" if invoke else "textFormat") in output_config + + +def _invoke_reply(answer: dict[str, JsonValue], native: bool) -> Reply: + block: Final[dict[str, JsonValue]] = ( + {"type": "text", "text": json.dumps(answer)} + if native + else {"type": "tool_use", "id": "toolu_scripted", "name": _JSON_TOOL, "input": answer} + ) + message: Final[dict[str, JsonValue]] = { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": [block], + "stop_reason": "end_turn" if native else "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + return Reply(body=json.dumps(message).encode()) + + +def _converse_reply(answer: dict[str, JsonValue], native: bool) -> Reply: + block: Final[dict[str, JsonValue]] = ( + {"text": json.dumps(answer)} + if native + else {"toolUse": {"toolUseId": "tooluse_scripted", "name": _JSON_TOOL, "input": answer}} + ) + converse: Final[dict[str, JsonValue]] = { + "output": {"message": {"role": "assistant", "content": [block]}}, + "stopReason": "end_turn" if native else "tool_use", + "usage": {"inputTokens": 11, "outputTokens": 7, "totalTokens": 18}, + "metrics": {"latencyMs": 1}, + } + return Reply(body=json.dumps(converse).encode()) + + +def _bedrock(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + found: Final = _MARKER.search(json.dumps(body)) + assert found is not None, body + invoke: Final = "/invoke" in request.target + reply: Final[Callable[[dict[str, JsonValue], bool], Reply]] = _invoke_reply if invoke else _converse_reply + return reply({"city": found.group(1)}, _native(body, invoke)) + + +@pytest.fixture +def opus_4_8_opted_in_for_the_process(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", _COST_MAP.validate_python(litellm.model_cost)) # pyright: ignore[reportUnknownMemberType] # the cost map is untyped + litellm.register_model( + {_OPUS_4_8: {"supports_bedrock_native_structured_output": True}}, persist_across_reloads=False + ) + yield + monkeypatch.undo() + _invalidate_model_cost_lowercase_map() + + +def _model(route: str) -> str: + return f"bedrock/{route}/{_OPUS_4_8}" + + +def _connection(wire: Wire) -> dict[str, JsonValue]: + return {"api_base": wire.url, "api_key": "synthetic-bedrock-bearer", "aws_region_name": "us-east-1"} + + +def _sent_native(wire: Wire, route: str, call: Callable[[list[dict[str, str]]], object]) -> bool: + marker_hex: Final = uuid.uuid4().hex + response: Final = call([{"role": "user", "content": f"Name a city. probe {marker_hex}"}]) + assert isinstance(response, ModelResponse), response + assert json.loads(str(response.choices[0].message.content)) == {"city": marker_hex}, response # pyright: ignore[reportAttributeAccessIssue] # choices also types streaming chunks + (request,) = wire.drain() + return _native(_JSON.validate_json(request.body), route == "invoke") + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +@pytest.mark.usefixtures("opus_4_8_opted_in_for_the_process") +def test_a_register_model_native_output_opt_in_reaches_every_call_but_a_deployment_that_opts_out(route: str) -> None: + with wire_server(_bedrock) as wire: + router: Final = Router( + model_list=[ + { + "model_name": "opted-out", + "litellm_params": {"model": _model(route), **_connection(wire)}, + "model_info": {"supports_bedrock_native_structured_output": False}, + }, + {"model_name": "inherits", "litellm_params": {"model": _model(route), **_connection(wire)}}, + ], + num_retries=0, + ) + assert _sent_native( + wire, + route, + lambda messages: router.completion(model="inherits", messages=messages, **_OPTIONS), # pyright: ignore[reportUnknownMemberType] # Router.completion leaves **kwargs untyped + ) + assert not _sent_native( + wire, + route, + lambda messages: router.completion(model="opted-out", messages=messages, **_OPTIONS), # pyright: ignore[reportUnknownMemberType] # Router.completion leaves **kwargs untyped + ) + assert _sent_native( + wire, + route, + lambda messages: litellm.completion(_model(route), messages, **_connection(wire), **_OPTIONS), + ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7efcc76073f..8331fb1937b 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,11 +2,11 @@ import asyncio import base64 import importlib import os -from collections.abc import Coroutine, Iterator +from collections.abc import Callable, Coroutine, Iterator from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Final, Literal, TypeAlias import boto3 import httpx @@ -308,6 +308,22 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None + + +@pytest.fixture +def set_native_structured_output( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> Callable[[str, NativeStructuredOutputKey], None]: + def set_flags(model_cost_key: str, native_key: NativeStructuredOutputKey) -> None: + entry: Final = litellm.model_cost[model_cost_key] + monkeypatch.setitem(entry, "supports_native_structured_output", native_key == "legacy") + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key == "bedrock") + litellm.get_model_info.cache_clear() + + return set_flags + + @pytest.fixture def local_beta_headers_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 1fda656060a..c08eeea4bb9 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -4,6 +4,7 @@ import functools import json import os import sys +from collections.abc import Mapping from typing import Final, cast import pytest @@ -29,6 +30,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( parse_tool_call_arguments, responses_reasoning_items_from_thinking_blocks, salvage_concatenated_tool_arguments, + schema_grammar_branch_count, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_property_count, split_concatenated_json_objects, strip_encrypted_reasoning_from_messages, strip_litellm_response_only_fields, @@ -2362,3 +2367,75 @@ def test_strip_litellm_response_only_fields_drops_nested_tool_call_provider_fiel assert "provider_specific_fields" not in function assert function["name"] == "get_weather" assert function["arguments"] == '{"city": "Paris"}' + + +@pytest.mark.parametrize( + ("schema", "is_open"), + [ + ({"type": "object", "properties": {"a": {"type": "string"}}}, False), + ({"type": "object", "properties": {}}, False), + ({"type": "object", "additionalProperties": False}, False), + ({"type": "string"}, False), + ({"type": "object"}, True), + ({"type": ["object", "null"]}, True), + ({"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": True}, True), + ({"type": "object", "additionalProperties": {"type": "integer"}}, True), + ({"type": "object", "properties": {}, "patternProperties": {"^x": {"type": "string"}}}, True), + ({"type": "object", "properties": {"inner": {"anyOf": [{"type": "object"}, {"type": "null"}]}}}, True), + ({"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}, True), + ], +) +def test_schema_has_open_object_flags_any_node_that_accepts_unlisted_keys(schema, is_open): + assert schema_has_open_object(schema) is is_open + + +def _object_with(properties: Mapping[str, object], **defs: Mapping[str, object]) -> Mapping[str, object]: + return {"type": "object", "properties": properties, **defs} + + +@pytest.mark.parametrize( + ("schema", "unresolvable"), + [ + (_object_with({"a": {"type": "string"}}), False), + (_object_with({"a": {"$ref": "#/$defs/A"}, "b": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/definitions/A"}}, definitions={"A": {"$ref": "#/definitions/B"}, "B": {"type": "integer"}}), False), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"items": {"$ref": "/$defs/B"}}, "B": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {"A": {"type": "string"}}}), True), + (_object_with({"a": {"$ref": "other.json#/A"}}), True), + (_object_with({"parent": {"$ref": "#"}}), True), + (_object_with({"a": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "#/$defs/A"}}}}), True), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "/$defs/A"}}}}), True), + ( + _object_with( + {"a": {"$ref": "#/$defs/A"}}, + **{"$defs": {"A": {"properties": {"b": {"$ref": "#/$defs/B"}}}, "B": {"properties": {"a": {"$ref": "#/$defs/A"}}}}}, + ), + True, + ), + ], + ids=["no-refs", "shared-def", "chained-definitions", "root-path-defs", "missing-def", "external", "root", "self-cycle", "root-path-self-cycle", "mutual-cycle"], +) +def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, unresolvable): + assert schema_has_unresolvable_ref(schema) is unresolvable + + +@pytest.mark.parametrize( + ("schema", "properties", "branches"), + [ + (_object_with({"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, required=["a"]), 3, 8), + ( + _object_with( + {"x": {"$ref": "#/$defs/Pair"}, "y": {"$ref": "#/$defs/Pair"}}, + required=["x", "y"], + **{"$defs": {"Pair": _object_with({"l": {"type": "string"}, "r": {"type": "string"}}, required=["l"])}}, + ), + 4, + 7, + ), + (_object_with({"inner": _object_with({"z": {"type": "string"}})}, required=["inner"]), 2, 4), + ], + ids=["flat", "shared-def-counted-once", "nested"], +) +def test_schema_size_counts_every_written_object_once(schema, properties, branches): + assert schema_property_count(schema) == properties + assert schema_grammar_branch_count(schema) == branches diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index bf977325a85..ec6af7f27c3 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1892,6 +1892,25 @@ def test_bedrock_tools_unpack_defs(): _bedrock_tools_pt(tools=tools) +@pytest.mark.parametrize( + "parameters", + [ + {"type": ["object", "null"]}, + {"type": "object", "properties": {"tier": {"type": "integer"}}, "$defs": [{"type": "string"}]}, + ], + ids=["root_type_list", "defs_list"], +) +def test_bedrock_tools_pt_sends_malformed_parameters_as_an_object_schema(parameters: dict[str, object]) -> None: + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt + + tools: Final = [{"type": "function", "function": {"name": "json_tool_call", "parameters": parameters}}] + + input_schema: Final = _bedrock_tools_pt(tools=tools)[0]["toolSpec"]["inputSchema"]["json"] + + assert input_schema["type"] == "object" + assert "$defs" not in input_schema + + def test_bedrock_tools_pt_strict_parameter(): """Regression for strict tools on the Bedrock Converse path. diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 724e137c835..6323132db36 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1346,6 +1346,74 @@ def test_opus_uses_native_structured_output(model_name): assert optional_params.get("json_mode") is True +@pytest.mark.parametrize( + ("field_schema", "native"), + [ + ({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, True), + ({"type": "object", "additionalProperties": {"type": "integer"}}, False), + ({"anyOf": [{"type": "object"}, {"type": "null"}]}, False), + ], + ids=["closed-object", "dict-field", "nullable-dict-field"], +) +def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(field_schema, native): + optional_params: Final = AnthropicConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": {"type": "object", "properties": {"field": field_schema}, "required": ["field"]}, + }, + } + }, + optional_params={}, + model="claude-opus-4-8", + drop_params=False, + ) + + assert ("output_format" in optional_params) is native + assert ([tool["name"] for tool in optional_params.get("tools", [])] == ["json_tool_call"]) is not native + + + +@pytest.mark.parametrize( + ("model", "thinking_params"), + [ + ("claude-opus-4-8", {"thinking": {"type": "adaptive"}}), + ("claude-opus-4-8", {"reasoning_effort": "low"}), + ("claude-opus-4-8", {"reasoning_effort": "none"}), + ("claude-sonnet-5-5", {}), + ( + "claude-opus-4-8", + {"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]}, + ), + ], + ids=["adaptive-thinking", "reasoning-effort", "reasoning-effort-none", "model-without-forced-tools", "caller-tools"], +) +def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(model, thinking_params): + optional_params: Final = AnthropicConfig().map_openai_params( + non_default_params={ + **thinking_params, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": { + "type": "object", + "properties": {"field": {"type": "object", "additionalProperties": {"type": "integer"}}}, + "required": ["field"], + }, + }, + }, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + assert "output_format" in optional_params + assert "json_tool_call" not in [tool["name"] for tool in optional_params.get("tools", [])] + def test_native_structured_output_uses_bundled_capability_when_remote_map_lags( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -9377,17 +9445,26 @@ def test_map_openai_params_lets_a_native_anthropic_search_tool_replace_the_opena @pytest.mark.parametrize( - ("config", "model", "provider"), + ("config", "model", "provider", "model_cost_key"), [ - pytest.param(AmazonAnthropicClaudeConfig(), "us.anthropic.claude-sonnet-5-5", "bedrock", id="bedrock-invoke"), - pytest.param(VertexAIAnthropicConfig(), "claude-sonnet-5-5", "vertex_ai", id="vertex_ai"), + pytest.param( + AmazonAnthropicClaudeConfig(), + "us.anthropic.claude-sonnet-5-5", + "bedrock", + "us.anthropic.claude-sonnet-5-5", + id="bedrock-invoke", + ), + pytest.param( + VertexAIAnthropicConfig(), "claude-sonnet-5-5", "vertex_ai", "vertex_ai/claude-sonnet-5-5", id="vertex_ai" + ), ], ) def test_map_openai_params_judges_tool_search_on_the_real_model_when_tool_based_response_format_swaps_it( - config, model, provider + set_native_structured_output, config, model, provider, model_cost_key ): from litellm.responses.litellm_completion_transformation.tool_search import anthropic_tool_search_chat_tool + set_native_structured_output(model_cost_key, None) assert litellm.supports_tool_search(model, provider) mapped: Final = config.map_openai_params( non_default_params={ @@ -9549,6 +9626,17 @@ def test_json_mode_tool_use_turn_adds_no_stop_fields_to_the_message(): assert not message.provider_specific_fields +def test_json_mode_keeps_the_thinking_that_preceded_the_json_tool_call(): + thinking_block: Final = {"type": "thinking", "thinking": "The user wants a greeting.", "signature": "sig-1"} + body: Final = _json_mode_message_body("tool_use", None) + result: Final = _transform_json_mode_body({**body, "content": [thinking_block, *body["content"]]}) + + message: Final = result.choices[0].message + assert json.loads(message.content) == {"answer": "Hello"} + assert message.reasoning_content == thinking_block["thinking"] + assert [block["signature"] for block in message.thinking_blocks] == [thinking_block["signature"]] + + def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None: result = AnthropicConfig().transform_parsed_response( completion_response=MagicMock(), diff --git a/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py b/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py index b2850192f91..0ed3b907a6f 100644 --- a/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py +++ b/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py @@ -12,6 +12,18 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig class TestOutputFormatArrayObjectConstraints: + def test_closes_an_object_whose_type_is_a_union_with_null(self): + schema = { + "type": "object", + "properties": { + "address": {"type": ["object", "null"], "properties": {"city": {"type": "string"}}}, + }, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["address"]["additionalProperties"] is False + def test_removes_uniqueitems(self): schema = { "type": "object", @@ -129,7 +141,7 @@ class TestOutputFormatArrayObjectConstraints: schema = { "type": "array", "items": {"type": "string"}, - "minItems": 1, + "minItems": 2, "maxItems": 10, "uniqueItems": True, "minContains": 2, @@ -139,7 +151,7 @@ class TestOutputFormatArrayObjectConstraints: result = AnthropicConfig.filter_anthropic_output_schema(schema) assert result["description"] == ( - "Note: minimum number of items: 1, maximum number of items: 10, " + "Note: minimum number of items: 2, maximum number of items: 10, " "all array items must be unique, minimum number of matching items: 2, " "maximum number of matching items: 3." ) diff --git a/tests/unit/llms/anthropic/test_anthropic_schema_filter.py b/tests/unit/llms/anthropic/test_anthropic_schema_filter.py index 32423d6679f..407a52c183e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_schema_filter.py +++ b/tests/unit/llms/anthropic/test_anthropic_schema_filter.py @@ -69,7 +69,7 @@ class TestFilterAnthropicOutputSchema: schema = { "type": "array", "items": {"type": "string"}, - "minItems": 1, + "minItems": 2, "maxItems": 10, } @@ -80,9 +80,27 @@ class TestFilterAnthropicOutputSchema: assert result["type"] == "array" assert result["items"] == {"type": "string"} # Description should contain constraint info - assert "minimum number of items: 1" in result["description"] + assert "minimum number of items: 2" in result["description"] assert "maximum number of items: 10" in result["description"] + def test_keeps_min_items_of_zero_or_one(self): + for min_items in (0, 1): + schema = {"type": "array", "items": {"type": "string"}, "minItems": min_items} + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["minItems"] == min_items + assert "description" not in result + + def test_moves_a_min_items_that_is_not_an_integer_into_the_description(self): + for min_items in (True, 1.0): + schema = {"type": "array", "items": {"type": "string"}, "minItems": min_items} + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert "minItems" not in result + assert "minimum number of items" in result["description"] + def test_handles_nested_schemas(self): """Test that nested schemas are also filtered.""" schema = { @@ -100,7 +118,7 @@ class TestFilterAnthropicOutputSchema: } }, }, - "minItems": 1, + "minItems": 2, } }, } @@ -336,7 +354,7 @@ class TestFilterAnthropicOutputSchema: schema = { "type": "array", "items": {"type": "string"}, - "minItems": 1, + "minItems": 2, "maxItems": 10, "uniqueItems": True, "minContains": 2, @@ -346,7 +364,7 @@ class TestFilterAnthropicOutputSchema: result = AnthropicConfig.filter_anthropic_output_schema(schema) assert result["description"] == ( - "Note: minimum number of items: 1, maximum number of items: 10, " + "Note: minimum number of items: 2, maximum number of items: 10, " "all array items must be unique, minimum number of matching items: 2, " "maximum number of matching items: 3." ) diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 16f44f4f35f..ca34a4eee06 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -3,11 +3,13 @@ import copy import json import uuid from types import SimpleNamespace +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest +from pydantic import BaseModel, ConfigDict # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. @@ -482,7 +484,7 @@ def test_output_config_format_converted_for_bedrock_chat_invoke_request(): with patch( # test-quality-ok: pin non-native path "litellm.llms.bedrock.common_utils._bedrock_model_supports", - side_effect=lambda _model, key: key == "supports_output_config", + side_effect=lambda _model, key, *_: key == "supports_output_config", ): result = config.transform_request( model="anthropic.claude-opus-4-7", @@ -504,10 +506,12 @@ def test_output_config_format_converted_for_bedrock_chat_invoke_request(): @pytest.mark.parametrize("model", ["anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-8"]) -def test_output_config_format_inlined_for_bedrock_chat_invoke_opus_4_7_and_4_8(local_model_cost_map, model): - """Bedrock rejects ``output_config.format`` on Claude Opus 4.7 and 4.8, so the - Invoke chat path inlines the schema into the last user message and keeps effort, - driven by the cost map alone (no capability stub).""" +def test_output_config_format_inlined_for_bedrock_chat_invoke_profile_without_native_output( + set_native_structured_output, model +): + """On a profile without native structured output the Invoke chat path inlines the + schema into the last user message and keeps effort.""" + set_native_structured_output(model, None) schema = {"type": "object", "properties": {"answer": {"type": "string"}}} result = AmazonAnthropicClaudeConfig().transform_request( @@ -525,12 +529,155 @@ def test_output_config_format_inlined_for_bedrock_chat_invoke_opus_4_7_and_4_8(l assert json.loads(result["messages"][-1]["content"][-1]["text"]) == schema +TREE_SCHEMA: Final = { + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": { + "Node": { + "type": "object", + "properties": {"children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}}, + } + }, +} +OPEN_OBJECT_SCHEMA: Final = { + "type": "object", + "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, +} + + +def _invoke_params_for_schema( + model: str, schema: Mapping[str, object], thinking_params: Mapping[str, object] | None = None +) -> Mapping[str, object]: + return AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, + **({"thinking": {"type": "adaptive"}} if thinking_params is None else thinking_params), + }, + optional_params={}, + model=model, + drop_params=False, + ) + + +@pytest.mark.parametrize( + ("native_key", "thinking_params", "native"), + [ + ("legacy", {}, True), + ("bedrock", {}, False), + ("bedrock", {"thinking": {"type": "adaptive"}}, True), + ("bedrock", {"reasoning_effort": "low"}, True), + ], + ids=["legacy-key", "bedrock-key-no-thinking", "bedrock-key-adaptive", "bedrock-key-effort"], +) +def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( + set_native_structured_output, native_key, thinking_params, native +): + model = "us.anthropic.claude-haiku-5-5" + set_native_structured_output(model, native_key) + schema = {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]} + + result = _invoke_params_for_schema(model, schema, thinking_params) + + assert ("output_format" in result) is native + assert ("tools" in result) is not native + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +@pytest.mark.parametrize("schema", [TREE_SCHEMA, OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"]) +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( + set_native_structured_output, schema, native_key +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + result = _invoke_params_for_schema(model, schema) + + assert "output_format" not in result + assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_closed_schema_goes_native_on_either_native_key(set_native_structured_output, native_key): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + schema = {"type": "object", "properties": {"age": {"type": "integer", "minimum": 0}}, "required": ["age"]} + + result = _invoke_params_for_schema(model, schema) + + assert result["output_format"]["schema"]["properties"]["age"]["type"] == "integer" + assert "minimum" not in result["output_format"]["schema"]["properties"]["age"] + assert "tools" not in result + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_native_structured_output, native_key): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + party_ref = {"$ref": "#/$defs/Party"} + schema = { + "type": "object", + "properties": {"buyer": party_ref, "seller": party_ref}, + "required": ["buyer", "seller"], + "$defs": { + "Party": { + "type": "object", + "properties": {"email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + "required": ["email"], + } + }, + } + + result = _invoke_params_for_schema(model, schema) + + sent = result["output_format"]["schema"] + assert sent["properties"]["buyer"] == party_ref + assert json.dumps(sent).count('"anyOf"') == 1 + assert "tools" not in result + + + +class Relabel(BaseModel): + model_config = ConfigDict(frozen=True) + + box: str + new_label: str + + +class PuzzleAnswer(BaseModel): + model_config = ConfigDict(frozen=True) + + relabels: list[Relabel] + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve( + set_native_structured_output, native_key +): + model = "us.anthropic.claude-haiku-5-5" + set_native_structured_output(model, native_key) + config: Final = AmazonAnthropicClaudeConfig() + + result: Final = config.map_openai_params( + non_default_params={ + "response_format": config.get_json_schema_from_pydantic_object(PuzzleAnswer), + "thinking": {"type": "adaptive"}, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + sent: Final = result["output_format"]["schema"] + item_ref: Final = sent["properties"]["relabels"]["items"]["$ref"] + assert item_ref.removeprefix("#/$defs/") in sent["$defs"] + assert "tools" not in result + def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map): """Bedrock Invoke chat path forwards ``output_config.format`` alongside effort for models with native structured-output support (Claude Sonnet 4.6).""" schema_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}, "additionalProperties": False}, } result = AmazonAnthropicClaudeConfig().transform_request( @@ -671,7 +818,7 @@ def test_bedrock_chat_invoke_drop_params_keeps_native_output_config_format(local monkeypatch.setattr(litellm, "drop_params", True) schema_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}, "additionalProperties": False}, } result = AmazonAnthropicClaudeConfig().transform_request( @@ -713,10 +860,11 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode "model", ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], ) -def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): - """Regression: Bedrock rejects both native ``output_config.format`` and forced - tool_choice for Fable 5.1, so invoke must use the tool-based path without a +def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(set_native_structured_output, model): + """Regression: Bedrock rejects forced tool_choice for Fable 5.1, so on a profile + without native structured output invoke must use the tool-based path without a forced ``tool_choice``.""" + set_native_structured_output(model, None) result = AmazonAnthropicClaudeConfig().map_openai_params( non_default_params={ "response_format": { @@ -738,7 +886,10 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice @pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) -def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): +def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking( + set_native_structured_output, model +): + set_native_structured_output(model, None) result = AmazonAnthropicClaudeConfig().map_openai_params( non_default_params={ "response_format": { @@ -761,9 +912,10 @@ def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_th assert result["output_config"] == {"effort": "high"} -def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(local_model_cost_map): +def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(set_native_structured_output): """Anthropic accepts a forced tool_choice next to adaptive thinking, and dropping it lets the model answer in prose instead of the requested JSON.""" + set_native_structured_output("us.anthropic.claude-opus-5", None) result = AmazonAnthropicClaudeConfig().map_openai_params( non_default_params={ "response_format": { diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6c996a7ce75..0ed713f056e 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -3,7 +3,7 @@ import copy import json import os import re -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -12,12 +12,18 @@ import pytest import respx import litellm -from litellm import ModelResponse, completion +from litellm import ModelResponse, Router, completion from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, _bedrock_tools_pt, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE +from litellm.llms.anthropic.chat.transformation import ( + NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES, + NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, + NATIVE_OUTPUT_MAX_UNION_PARAMETERS, +) +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -1246,7 +1252,9 @@ async def test_transform_request_replays_redacted_thinking_only_to_claude( """A model id that names its model decides on its own. An application inference profile ARN hides its model, so the deployment's ``base_model`` is what marks it as Claude, and without one the redacted block is dropped the way it is for every non-Claude target.""" - redacted_data: Final = base64.b64encode(b"EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP").decode() + redacted_data: Final = base64.b64encode( + b"EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" + ).decode() messages: Final = [ {"role": "user", "content": "hi"}, { @@ -1475,32 +1483,25 @@ def test_output_config_format_dropped_on_unsupported_converse_model_warns(caplog """When Converse model lacks native structured-output support, the silently dropped ``output_config.format`` must surface as a warning so callers can diagnose plain-text responses.""" - from unittest.mock import patch - config = AmazonConverseConfig() schema = { "type": "object", "properties": {"answer": {"type": "string"}}, } - with patch.object( - AmazonConverseConfig, - "_supports_native_structured_outputs", - return_value=False, - ): - with caplog.at_level("WARNING"): - result = config._transform_request( - model="bedrock/converse/us.anthropic.claude-3-haiku-20240307-v1:0", - messages=[{"role": "user", "content": "hi"}], - optional_params={ - "maxTokens": 256, - "output_config": { - "format": {"type": "json_schema", "schema": schema}, - }, + with caplog.at_level("WARNING"): + result = config._transform_request( + model="bedrock/converse/us.anthropic.claude-3-haiku-20240307-v1:0", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "maxTokens": 256, + "output_config": { + "format": {"type": "json_schema", "schema": schema}, }, - litellm_params={}, - headers={}, - ) + }, + litellm_params={}, + headers={}, + ) assert "outputConfig" not in result assert any("dropping `output_config.format`" in record.getMessage() for record in caplog.records) @@ -4841,24 +4842,6 @@ def test_translate_response_format_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -BEDROCK_OPUS_4_7_AND_4_8_MODELS: Final = ( - "anthropic.claude-opus-4-7", - "global.anthropic.claude-opus-4-7", - "us.anthropic.claude-opus-4-7", - "eu.anthropic.claude-opus-4-7", - "au.anthropic.claude-opus-4-7", - "jp.anthropic.claude-opus-4-7", - "anthropic.claude-opus-4-8", - "global.anthropic.claude-opus-4-8", - "us.anthropic.claude-opus-4-8", - "eu.anthropic.claude-opus-4-8", - "au.anthropic.claude-opus-4-8", - "jp.anthropic.claude-opus-4-8", - "us-gov.anthropic.claude-opus-4-8", - "us-gov-west-1/anthropic.claude-opus-4-8", - "us-gov-east-1/anthropic.claude-opus-4-8", -) - CAPITAL_RESPONSE_FORMAT: Final = { "type": "json_schema", "json_schema": { @@ -4873,10 +4856,15 @@ CAPITAL_RESPONSE_FORMAT: Final = { } -def _converse_request_for_json_schema(model: str, stream: bool) -> tuple[dict, dict]: +ADAPTIVE_THINKING: Final = {"type": "adaptive"} + + +def _converse_request_for_json_schema( + model: str, stream: bool, thinking_params: Mapping[str, object] | None = None +) -> tuple[Mapping[str, object], RequestObject]: config = AmazonConverseConfig() optional_params = config.map_openai_params( - non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "stream": stream}, + non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "stream": stream, **(thinking_params or {})}, optional_params={}, model=model, drop_params=False, @@ -4891,21 +4879,452 @@ def _converse_request_for_json_schema(model: str, stream: bool) -> tuple[dict, d return optional_params, request -@pytest.mark.parametrize("model", BEDROCK_OPUS_4_7_AND_4_8_MODELS) +@pytest.mark.parametrize( + ("native_profile", "tool_profile"), + [ + ("us.anthropic.claude-opus-4-8", "anthropic.claude-opus-4-8"), + ("global.anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7"), + ("anthropic.claude-opus-4-8", "us-gov.anthropic.claude-opus-4-8"), + ], +) @pytest.mark.parametrize("stream", [False, True]) -def test_opus_4_7_and_4_8_json_schema_sent_as_forced_tool_not_output_config(monkeypatch, model, stream): - """Regression for issue #27846: Bedrock rejects outputConfig on Opus 4.7 and 4.8 - (``output_config.format: Extra inputs are not permitted``), so json_schema has to - go out as the forced json_tool_call tool, streamed through fake_stream.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) +def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag( + set_native_structured_output, native_profile, tool_profile, stream +): + set_native_structured_output(native_profile, "bedrock") + set_native_structured_output(tool_profile, None) - optional_params, request = _converse_request_for_json_schema(model=model, stream=stream) + thinking_params: Final = {"thinking": ADAPTIVE_THINKING} + native_params, native_request = _converse_request_for_json_schema(native_profile, stream, thinking_params) + tool_params, tool_request = _converse_request_for_json_schema(tool_profile, stream, thinking_params) + + assert native_request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "capital" + assert "toolConfig" not in native_request + assert "fake_stream" not in native_params + assert "outputConfig" not in tool_request + assert [tool["toolSpec"]["name"] for tool in tool_request["toolConfig"]["tools"]] == ["json_tool_call"] + assert tool_request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert tool_params.get("fake_stream", False) is stream + + +@pytest.mark.parametrize( + ("native_key", "thinking_params", "native"), + [ + ("legacy", {}, True), + ("bedrock", {}, False), + ("bedrock", {"thinking": ADAPTIVE_THINKING}, True), + ("bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ("bedrock", {"reasoning_effort": "low"}, True), + ("bedrock", {"reasoning_effort": "none"}, False), + ("bedrock", {"thinking": {"type": "disabled"}}, False), + ], + ids=[ + "legacy-key", + "bedrock-key-no-thinking", + "bedrock-key-adaptive", + "bedrock-key-enabled", + "bedrock-key-effort", + "bedrock-key-effort-none", + "bedrock-key-thinking-disabled", + ], +) +def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( + set_native_structured_output, native_key, thinking_params, native +): + model: Final = "us.anthropic.claude-haiku-5-5" + set_native_structured_output(model, native_key) + + _, request = _converse_request_for_json_schema(model, False, thinking_params) + + assert ("outputConfig" in request) is native + assert ("toolConfig" in request) is not native + + +def test_adaptive_thinking_is_kept_next_to_json_schema_on_a_native_output_profile(set_native_structured_output): + model: Final = "us.anthropic.claude-opus-5" + set_native_structured_output(model, "bedrock") + config: Final = AmazonConverseConfig() + thinking: Final = {"type": "adaptive", "display": "summarized"} + + optional_params: Final = config.map_openai_params( + non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": thinking}, + optional_params={}, + model=model, + drop_params=False, + ) + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["additionalModelRequestFields"]["thinking"] == thinking + assert request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "capital" + assert "toolConfig" not in request + + +_OPUS_4_8_INVOKE_RESPONSE: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": '{"city": "Paris", "country": "France"}'}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, +} + + +def _routed_request_body(router: Router, model_name: str, route: str) -> Mapping[str, object]: + response: Final = _mock_converse_response() + if route == "invoke": + response.json.return_value = _OPUS_4_8_INVOKE_RESPONSE + response.text = json.dumps(_OPUS_4_8_INVOKE_RESPONSE) + client: Final = HTTPHandler() + with patch.object(client, "post", return_value=response) as mock_post: + router.completion( + model=model_name, + messages=[{"role": "user", "content": "Name the capital of France."}], + response_format=CAPITAL_RESPONSE_FORMAT, + thinking=ADAPTIVE_THINKING, + client=client, + ) + return json.loads(mock_post.call_args.kwargs["data"]) + + +def _sends_native_output(body: Mapping[str, object], route: str) -> bool: + if route == "converse": + return "outputConfig" in body and "toolConfig" not in body + output_config: Final = body.get("output_config") + return isinstance(output_config, dict) and "format" in output_config and "tools" not in body + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_a_deployment_native_output_opt_in_stays_on_that_deployment(set_native_structured_output, route): + set_native_structured_output("us.anthropic.claude-opus-4-8", None) + litellm_params: Final = { + "model": f"bedrock/{route}/us.anthropic.claude-opus-4-8", + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + } + router: Final = Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": litellm_params, + "model_info": {"supports_bedrock_native_structured_output": True}, + }, + {"model_name": "sibling", "litellm_params": litellm_params}, + ] + ) + + assert _sends_native_output(_routed_request_body(router, "opted-in", route), route) + assert not _sends_native_output(_routed_request_body(router, "sibling", route), route) + + +NATIVE_OUTPUT_REJECTED_KEYWORDS: Final = frozenset({"minimum", "maximum", "uniqueItems", "oneOf"}) +CONSTRAINED_PERSON_SCHEMA: Final = { + "type": "object", + "properties": { + "age": {"type": "integer", "minimum": 0, "maximum": 150}, + "tags": {"type": "array", "items": {"type": "string"}, "minItems": 2, "uniqueItems": True}, + "aliases": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "contact": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, + "address": {"type": ["object", "null"], "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, + "required": ["age", "tags", "aliases", "contact", "address"], +} +RECURSIVE_TREE_SCHEMA: Final = { + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "required": ["root"], + "$defs": { + "Node": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}, + }, + "required": ["name", "children"], + } + }, +} +OPEN_OBJECT_SCHEMA: Final = { + "type": "object", + "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, + "required": ["counts"], +} +LOOKAROUND_PATTERN_SCHEMA: Final = { + "type": "object", + "properties": {"amount": {"type": "string", "pattern": "^(?!^[-+.]*$)[+-]?0*\\d*\\.?\\d*$"}}, + "required": ["amount"], +} +UNKNOWN_FORMAT_SCHEMA: Final = { + "type": "object", + "properties": {"blob": {"type": "string", "format": "x-litellm-unlisted-format"}}, + "required": ["blob"], +} +ROOT_REF_SCHEMA: Final = { + "type": "object", + "properties": {"name": {"type": "string"}, "parent": {"anyOf": [{"$ref": "#"}, {"type": "null"}]}}, + "required": ["name", "parent"], +} + + +def _schema_keys(node: object) -> frozenset[str]: + if isinstance(node, dict): + return frozenset(node).union(*(_schema_keys(value) for value in node.values())) + if isinstance(node, list): + return frozenset().union(*(_schema_keys(item) for item in node)) + return frozenset() + + +def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> RequestObject: + config: Final = AmazonConverseConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, + "thinking": ADAPTIVE_THINKING, + }, + optional_params={}, + model=model, + drop_params=False, + ) + return config._transform_request( + model=model, + messages=[{"role": "user", "content": "Fill in a plausible example."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( + set_native_structured_output, native_key +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = _converse_request_for_schema(model, CONSTRAINED_PERSON_SCHEMA) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert not _schema_keys(sent) & NATIVE_OUTPUT_REJECTED_KEYWORDS + assert sent["additionalProperties"] is False + assert sent["properties"]["contact"]["anyOf"] == [{"type": "string"}, {"type": "integer"}] + assert "minimum value: 0" in sent["properties"]["age"]["description"] + assert "minItems" not in sent["properties"]["tags"] + assert sent["properties"]["aliases"]["minItems"] == 1 + assert sent["properties"]["address"]["additionalProperties"] is False + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +@pytest.mark.parametrize( + ("schema", "kept_keyword"), + [ + (RECURSIVE_TREE_SCHEMA, "$ref"), + (ROOT_REF_SCHEMA, "$ref"), + (OPEN_OBJECT_SCHEMA, "additionalProperties"), + (LOOKAROUND_PATTERN_SCHEMA, "pattern"), + (UNKNOWN_FORMAT_SCHEMA, "format"), + ], + ids=["recursive", "root-ref", "open-object", "lookaround-pattern", "unlisted-format"], +) +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( + set_native_structured_output, schema, kept_keyword, native_key +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = _converse_request_for_schema(model, schema) assert "outputConfig" not in request - assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] - assert request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} - assert optional_params.get("fake_stream", False) is stream + tool_spec: Final = request["toolConfig"]["tools"][0]["toolSpec"] + assert tool_spec["name"] == "json_tool_call" + assert kept_keyword in _schema_keys(tool_spec["inputSchema"]["json"]) + + +def _shared_defs_schema(defs_key: str) -> Mapping[str, object]: + party_ref: Final = {"$ref": f"#/{defs_key}/Party"} + return { + "type": "object", + "properties": {"buyer": party_ref, "seller": party_ref, "broker": {"anyOf": [party_ref, {"type": "null"}]}}, + "required": ["buyer", "seller", "broker"], + defs_key: { + "Party": { + "type": ["object", "null"], + "properties": {"name": {"type": "string"}, "email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + "required": ["name", "email"], + } + }, + } + + +@pytest.mark.parametrize("defs_key", ["$defs", "definitions"]) +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use( + set_native_structured_output, native_key, defs_key +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = _converse_request_for_schema(model, _shared_defs_schema(defs_key)) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert sent["properties"]["buyer"] == {"$ref": f"#/{defs_key}/Party"} + assert json.dumps(sent).count('"anyOf"') == 2 + assert sent[defs_key]["Party"]["additionalProperties"] is False + + +def _flat_schema(property_schema: Mapping[str, object], count: int, required: bool) -> Mapping[str, object]: + names: Final = tuple(f"field_{index}" for index in range(count)) + return { + "type": "object", + "properties": dict.fromkeys(names, property_schema), + "required": list(names) if required else [], + } + + +def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, object]: + groups: Final = { + f"group_{start}": _flat_schema({"type": "string"}, min(group_size, count - start), required=False) + for start in range(0, count, group_size) + } + return {"type": "object", "properties": groups, "required": list(groups)} + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +@pytest.mark.parametrize( + ("schema_for", "limit"), + [ + (lambda count: _flat_schema({"type": ["string", "null"]}, count, True), NATIVE_OUTPUT_MAX_UNION_PARAMETERS), + ( + lambda count: _flat_schema({"anyOf": [{"type": "string"}, {"type": "integer"}]}, count, True), + NATIVE_OUTPUT_MAX_UNION_PARAMETERS, + ), + (_optional_groups_schema, NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS), + ( + lambda count: _flat_schema({"type": "string"}, count, False), + NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES.bit_length() - 1, + ), + ], + ids=["type-list-unions", "any-of-unions", "optional", "optional-grammar-branches"], +) +def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( + set_native_structured_output, native_key, schema_for, limit +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + at_limit: Final = _converse_request_for_schema(model, schema_for(limit)) + past_limit: Final = _converse_request_for_schema(model, schema_for(limit + 1)) + + sent: Final = json.loads(at_limit["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert sent["properties"].keys() == schema_for(limit)["properties"].keys() + assert "outputConfig" not in past_limit + assert past_limit["toolConfig"]["tools"][0]["toolSpec"]["name"] == "json_tool_call" + + +@pytest.mark.parametrize( + ("native_key", "native_past_limit"), [("legacy", True), ("bedrock", False)], ids=["legacy-key", "bedrock-key"] +) +def test_field_count_limit_sends_the_json_tool_only_on_the_bedrock_native_key( + set_native_structured_output, native_key, native_past_limit +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + at_limit: Final = _converse_request_for_schema( + model, _flat_schema({"type": "string"}, BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES, True) + ) + past_limit: Final = _converse_request_for_schema( + model, _flat_schema({"type": "string"}, BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + 1, True) + ) + + assert "outputConfig" in at_limit + assert ("outputConfig" in past_limit) is native_past_limit + + +NUMERIC_FORMAT_SCHEMA: Final = { + "type": "object", + "properties": { + "count": {"type": "integer", "format": "int32"}, + "total": {"type": "integer", "format": "int64"}, + "ratio": {"type": "number", "format": "double"}, + }, + "required": ["count", "total", "ratio"], +} + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_numeric_formats_keep_native_output(set_native_structured_output, native_key): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = _converse_request_for_schema(model, NUMERIC_FORMAT_SCHEMA) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert {name: field["format"] for name, field in sent["properties"].items()} == { + "count": "int32", + "total": "int64", + "ratio": "double", + } + + +def _doubling_defs_schema(depth: int) -> Mapping[str, object]: + levels: Final = { + f"Level{index}": { + "type": "object", + "properties": { + "left": {"$ref": f"#/$defs/Level{index + 1}"}, + "right": {"$ref": f"#/$defs/Level{index + 1}"}, + }, + "required": ["left", "right"], + } + for index in range(depth) + } + return { + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Level0"}}, + "required": ["root"], + "$defs": {**levels, f"Level{depth}": {"type": "string"}}, + } + + +@pytest.mark.timeout(30) +def test_defs_that_double_at_every_level_reach_native_output_without_being_expanded(set_native_structured_output): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + schema: Final = _doubling_defs_schema(30) + + request: Final = _converse_request_for_schema(model, schema) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert sent["$defs"].keys() == schema["$defs"].keys() + assert sent["properties"]["root"] == {"$ref": "#/$defs/Level0"} + + +@pytest.mark.parametrize( + ("native_key", "forwarded"), [("legacy", True), ("bedrock", False)], ids=["legacy-key", "bedrock-key"] +) +def test_recursive_output_config_format_follows_the_profile_native_key( + set_native_structured_output, native_key, forwarded +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = AmazonConverseConfig()._transform_request( + model=model, + messages=[{"role": "user", "content": "Fill in a plausible example."}], + optional_params={"output_config": {"format": {"type": "json_schema", "schema": RECURSIVE_TREE_SCHEMA}}}, + litellm_params={}, + headers={}, + ) + + assert ("outputConfig" in request) is forwarded + def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): @@ -8008,10 +8427,13 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], ) -def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(local_model_cost_map, model): - """Regression: Bedrock rejects both ``outputConfig`` structured output and forced - tool_choice for Fable 5.1, so response_format must map to a tool without a forced - tool_choice.""" +def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( + set_native_structured_output, model +): + """Regression: Bedrock rejects forced tool_choice for Fable 5.1, so on a profile + without native structured output response_format must map to a tool without a + forced tool_choice.""" + set_native_structured_output(model, None) config = AmazonConverseConfig() result = config.map_openai_params( diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index c91eb75dbd9..eb6b4fdb063 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1050,6 +1050,7 @@ def test_bedrock_messages_forwards_output_config_with_output_format(): "schema": { "type": "object", "properties": {"answer": {"type": "string"}}, + "additionalProperties": False, }, } optional_params = { @@ -1098,7 +1099,7 @@ def test_bedrock_messages_converts_output_config_format_to_inline_schema(): with patch( "litellm.llms.bedrock.common_utils._bedrock_model_supports", - side_effect=lambda _model, key: key == "supports_output_config", + side_effect=lambda _model, key, *_: key == "supports_output_config", ): result = cfg.transform_anthropic_messages_request( model="anthropic.claude-opus-4-7", @@ -1170,7 +1171,7 @@ def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema with patch( "litellm.llms.bedrock.common_utils._bedrock_model_supports", - side_effect=lambda _model, key: key == "supports_output_config", + side_effect=lambda _model, key, *_: key == "supports_output_config", ): result = cfg.transform_anthropic_messages_request( model="anthropic.claude-opus-4-7", @@ -1677,7 +1678,12 @@ def test_bedrock_messages_does_not_add_dangerous_tool_use_beta_without_safeguard assert "dangerous-tool-use-2026-09-03" not in result.get("anthropic_beta", []) -_NAME_SCHEMA: Final = {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]} +_NAME_SCHEMA: Final = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "additionalProperties": False, +} @pytest.mark.parametrize( @@ -3216,11 +3222,12 @@ def test_bedrock_messages_forwards_output_config_format_natively(local_model_cos assert "zebra_count" not in json.dumps(result["messages"]) -def test_bedrock_messages_inlines_schema_for_claude_5(local_model_cost_map): - """Bedrock rejects ``output_config.format`` for the Claude 5 family, so the - schema falls back to the inline-text path instead of a deterministic 400.""" +def test_bedrock_messages_inlines_schema_for_a_profile_without_native_output(set_native_structured_output): + """On a profile without native structured output the schema falls back to the + inline-text path instead of a deterministic 400.""" from litellm.types.router import GenericLiteLLMParams + set_native_structured_output("us.anthropic.claude-sonnet-5", None) cfg = AmazonAnthropicClaudeMessagesConfig() schema = { "type": "object", @@ -3243,6 +3250,115 @@ def test_bedrock_messages_inlines_schema_for_claude_5(local_model_cost_map): assert json.loads(last_content[-1]["text"]) == schema +def _bedrock_messages_request_with_schema(model: str, schema: Mapping[str, object]) -> Mapping[str, object]: + from litellm.types.router import GenericLiteLLMParams + + return AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "say hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 100, + "output_config": {"format": {"type": "json_schema", "schema": schema}}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +def test_bedrock_messages_native_output_drops_schema_keywords_bedrock_rejects(set_native_structured_output): + model: Final = "us.anthropic.claude-sonnet-5" + set_native_structured_output(model, "bedrock") + schema: Final = { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": True}, + }, + "required": ["count", "ids"], + } + + result: Final = _bedrock_messages_request_with_schema(model, schema) + + sent: Final = result["output_config"]["format"]["schema"] + assert "minimum" not in sent["properties"]["count"] + assert "uniqueItems" not in sent["properties"]["ids"] + assert sent["additionalProperties"] is False + + +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_bedrock_messages_shared_defs_reach_native_output_once_instead_of_inlined_per_use( + set_native_structured_output, native_key +): + model: Final = "us.anthropic.claude-sonnet-5" + set_native_structured_output(model, native_key) + party_ref: Final = {"$ref": "#/$defs/Party"} + schema: Final = { + "type": "object", + "properties": {"buyer": party_ref, "seller": party_ref}, + "required": ["buyer", "seller"], + "additionalProperties": False, + "$defs": { + "Party": { + "type": "object", + "properties": {"email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + "required": ["email"], + "additionalProperties": False, + } + }, + } + + result: Final = _bedrock_messages_request_with_schema(model, schema) + + sent: Final = result["output_config"]["format"]["schema"] + assert sent["properties"]["buyer"] == party_ref + assert json.dumps(sent).count('"anyOf"') == 1 + + +MESSAGES_TREE_SCHEMA: Final = { + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": { + "Node": { + "type": "object", + "properties": {"children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}}, + } + }, +} +MESSAGES_OPEN_OBJECT_SCHEMA: Final = { + "type": "object", + "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, +} + + +@pytest.mark.parametrize( + "schema", [MESSAGES_TREE_SCHEMA, MESSAGES_OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"] +) +def test_bedrock_messages_schema_native_output_cannot_hold_is_inlined_on_a_bedrock_native_profile( + set_native_structured_output, schema +): + model: Final = "us.anthropic.claude-sonnet-5" + set_native_structured_output(model, "bedrock") + + result: Final = _bedrock_messages_request_with_schema(model, schema) + + assert "output_config" not in result + assert json.loads(result["messages"][-1]["content"][-1]["text"]) == schema + + +@pytest.mark.parametrize( + "schema", [MESSAGES_TREE_SCHEMA, MESSAGES_OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"] +) +def test_bedrock_messages_schema_native_output_cannot_hold_is_forwarded_on_a_legacy_native_profile( + set_native_structured_output, schema +): + model: Final = "us.anthropic.claude-sonnet-4-6" + set_native_structured_output(model, "legacy") + + result: Final = _bedrock_messages_request_with_schema(model, schema) + + assert result["output_config"]["format"]["schema"] == schema + + def test_bedrock_messages_legacy_output_format_wins_over_output_config_format(local_model_cost_map): """When a request carries both schema forms, the legacy top-level ``output_format`` keeps winning, matching the pre-existing precedence.""" @@ -3251,7 +3367,11 @@ def test_bedrock_messages_legacy_output_format_wins_over_output_config_format(lo cfg = AmazonAnthropicClaudeMessagesConfig() legacy_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"legacy_field": {"type": "string"}}}, + "schema": { + "type": "object", + "properties": {"legacy_field": {"type": "string"}}, + "additionalProperties": False, + }, } newer_format = { "type": "json_schema", @@ -3285,7 +3405,7 @@ def test_bedrock_messages_drop_params_keeps_native_output_config_format(local_mo cfg = AmazonAnthropicClaudeMessagesConfig() schema_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}, "additionalProperties": False}, } result = cfg.transform_anthropic_messages_request( @@ -3310,7 +3430,7 @@ def test_bedrock_messages_strips_effort_but_keeps_format_for_sonnet_4_5(local_mo cfg = AmazonAnthropicClaudeMessagesConfig() schema_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}}, + "schema": {"type": "object", "properties": {"zebra_count": {"type": "integer"}}, "additionalProperties": False}, } result = cfg.transform_anthropic_messages_request( diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 303f752ab0d..9aab5052cfe 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -659,8 +659,14 @@ def test_apply_structured_output_prefers_legacy_output_format(local_model_cost_m apply_bedrock_invoke_structured_output, ) - legacy = {"type": "json_schema", "schema": {"type": "object", "properties": {"a": {"type": "string"}}}} - newer = {"type": "json_schema", "schema": {"type": "object", "properties": {"b": {"type": "string"}}}} + legacy = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": False}, + } + newer = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"b": {"type": "string"}}, "additionalProperties": False}, + } body = { "messages": [{"role": "user", "content": "hi"}], "output_format": legacy, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 632ab020409..87f9d58e247 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1256,6 +1256,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_native_streaming": {"type": "boolean"}, "supports_image_size": {"type": "boolean"}, "supports_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output": {"type": "boolean"}, "use_openai_responses_path": {"type": "boolean"}, "off_peak_pricing": { "type": "object", diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index 09c9079d89e..bfaac44a483 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -224,9 +224,20 @@ def test_regional_bedrock_alias_requires_canonical_converse_metadata( _enforce_bedrock_converse_models(model_cost, ()) +GOVCLOUD_LAGGING_CAPABILITIES: Final = frozenset({"supports_bedrock_native_structured_output"}) + + +def _govcloud_may_lag(model_key: str, capability: str) -> bool: + is_govcloud: Final = model_key.startswith("us-gov.") or any( + part.startswith("us-gov-") for part in model_key.split("/") + ) + return is_govcloud and capability in GOVCLOUD_LAGGING_CAPABILITIES + + def test_get_model_info_bedrock_models(): """ Check for drift in base model info for bedrock models and regional model info for bedrock models. + GovCloud regions may lag their commercial base on Bedrock native structured output, never lead it. """ from litellm.llms.bedrock.common_utils import BedrockModelInfo @@ -261,6 +272,9 @@ def test_get_model_info_bedrock_models(): assert ( base_model_key in v ), f"{base_model_key} is not in model cost map for {k}" + if _govcloud_may_lag(k, base_model_key): + assert base_model_value or not v[base_model_key], f"{base_model_key} is on {k} but not its base" + continue assert ( v[base_model_key] == base_model_value ), f"{base_model_key} is not equal to {base_model_value} for model {k}" @@ -297,6 +311,8 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): """ Cross-region inference profiles carry litellm_provider "bedrock_converse", so the regional drift check above (which filters on "bedrock") never reaches them. + GovCloud is a separate AWS partition that gets features later, so a us-gov profile + may lack Bedrock native structured output its commercial base has, never the reverse. """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -318,6 +334,9 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): if not cap.startswith("supports_"): continue assert cap in v, f"{cap} is on {base_model_key} but missing from {k}" + if _govcloud_may_lag(k, cap): + assert base_value or not v[cap], f"{cap} is on {k} but not on {base_model_key}" + continue assert ( v[cap] == base_value ), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}"