diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..7c3a822a10c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2,7 +2,7 @@ import copy import json import re import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast @@ -27,15 +27,12 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, - is_object_schema, sanitize_input_schema_for_anthropic, - schema_fragment_ref, schema_grammar_branch_count, schema_has_lookaround_pattern, schema_has_open_object, schema_has_unresolvable_ref, schema_optional_property_count, - schema_string_formats, schema_union_property_count, unpack_defs, ) @@ -48,6 +45,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.llms.anthropic.output_schema import ( + shape_claude_output_schema, +) from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, @@ -166,9 +166,6 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( - ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") -) NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16 NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048 @@ -268,31 +265,6 @@ class _AnthropicToolResultBlock(TypedDict, total=False): content: ReadOnly[object] -_ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType( - { - "null": lambda v: v is None, - "boolean": lambda v: isinstance(v, bool), - "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), - "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), - "string": lambda v: isinstance(v, str), - "array": lambda v: isinstance(v, list), - "object": lambda v: isinstance(v, dict), - } -) - - -def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool: - """Whether ``schema``'s ``enum`` cannot match its declared ``type``.""" - enum_values: Final = schema.get("enum") - declared_type: Final = schema.get("type") - if not isinstance(enum_values, list) or declared_type is None: - return False - if isinstance(declared_type, list): - return True - check: Final = _ENUM_TYPE_CHECKS.get(declared_type) - return check is not None and not all(check(value) for value in enum_values) - - # Single, internal-only key on ``litellm_params`` used to thread the per- # request reverse map (sanitized -> original) from request build to response # parsing. ``litellm_params`` is never serialized to a provider; ``optional_ @@ -766,149 +738,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return params @staticmethod - def filter_anthropic_output_schema(schema: dict[str, Any]) -> dict[str, Any]: - """ - Filter out unsupported fields from JSON schema for Anthropic's output_format API. - - Anthropic's output_format doesn't support certain JSON schema properties. - These are constraints that cannot be enforced by the constrained-decoding - grammar Anthropic compiles the schema into, so the API rejects them with a - 400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type, - property 'uniqueItems' is not supported"): - - maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints - - minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints - - minLength/maxLength: string constraints - - minProperties/maxProperties/patternProperties/propertyNames: object constraints - - dependentRequired/dependentSchemas/unevaluatedProperties: object constraints - - if/then/else/not: conditional and negation keywords - - ``oneOf`` is also rejected ("Schema type 'oneOf' is not supported") and is - rewritten to ``anyOf``, matching the Anthropic SDK. Unknown keywords are - ignored by the API, so anything not listed here passes through untouched. - - This mirrors the transformation done by the Anthropic Python SDK. - See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works - - The SDK approach: - 1. Remove unsupported constraints from schema - 2. Add constraint info to description (e.g., "Must be at least 100") - 3. Validate responses against original schema - Args: - schema: The JSON schema dictionary to filter - - Returns: - A new dictionary with unsupported fields removed and descriptions updated - - Related issues: - - https://github.com/BerriAI/litellm/issues/19444 - """ - if not isinstance(schema, dict): - return schema - - constraint_labels: Final = { - "minItems": "minimum number of items: {}", - "maxItems": "maximum number of items: {}", - "uniqueItems": "all array items must be unique", - "contains": "array must contain an item matching: {}", - "minContains": "minimum number of matching items: {}", - "maxContains": "maximum number of matching items: {}", - "prefixItems": "leading items must match, in order: {}", - "minimum": "minimum value: {}", - "maximum": "maximum value: {}", - "exclusiveMinimum": "exclusive minimum value: {}", - "exclusiveMaximum": "exclusive maximum value: {}", - "multipleOf": "must be a multiple of {}", - "minLength": "minimum length: {}", - "maxLength": "maximum length: {}", - "minProperties": "minimum number of properties: {}", - "maxProperties": "maximum number of properties: {}", - "patternProperties": "properties whose names match each pattern must satisfy: {}", - "propertyNames": "property names must satisfy: {}", - "dependentRequired": "dependent required properties: {}", - "dependentSchemas": "dependent schemas: {}", - "unevaluatedProperties": "unevaluated properties must satisfy: {}", - "if": "conditional (if): {}", - "then": "conditional (then): {}", - "else": "conditional (else): {}", - "not": "must not match: {}", - } - schema_fields: Final[Mapping[str, object]] = MappingProxyType(schema) - min_items: Final = schema.get("minItems") - enforced_min_items: Final = ( - isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1) - ) - unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set()) - - # Build description additions from removed constraints. Iterating - # constraint_labels (not the set) keeps the note order deterministic across - # processes, so identical requests serialize identically regardless of - # PYTHONHASHSEED and stay cache-friendly. - constraint_descriptions: Final[list] = [] - for field, label in constraint_labels.items(): - if field not in schema or field not in unsupported_fields: - continue - value = schema_fields[field] - # A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no - # real requirement, so don't add a misleading advisory note for it. - if isinstance(value, bool) and not value: - continue - # Sub-schema constraints (e.g. ``contains``) are serialized as JSON so - # the advisory note preserves what the constraint actually required, - # instead of just noting that it existed. - note_value = json.dumps(value) if isinstance(value, (dict, list)) else value - constraint_descriptions.append(label.format(note_value)) - - result: Final[dict[str, Any]] = {} - - # Update description with removed constraint info - if constraint_descriptions: - existing_desc: Final = schema.get("description", "") - constraint_note: Final = "Note: " + ", ".join(constraint_descriptions) + "." - if existing_desc: - result["description"] = existing_desc + " " + constraint_note - else: - result["description"] = constraint_note - - drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema) - - for key, value in schema_fields.items(): - if key in unsupported_fields: - continue - if key == "type" and drops_conflicting_type: - continue - if key == "description" and "description" in result: - # Already handled above - continue - - if key in ("properties", "$defs", "definitions") and isinstance(value, dict): - nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict - result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()} - elif key == "items" and isinstance(value, dict): - nested_schema: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict - result[key] = AnthropicConfig.filter_anthropic_output_schema(nested_schema) - elif key == "anyOf" and isinstance(value, list): - result["anyOf"] = result.get("anyOf", []) + [ - AnthropicConfig.filter_anthropic_output_schema(item) for item in value - ] - elif key == "allOf" and isinstance(value, list): - result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value] - elif key == "$ref" and isinstance(value, str): - result[key] = schema_fragment_ref(value) - elif key == "oneOf" and isinstance(value, list): - # Anthropic rejects oneOf ("Schema type 'oneOf' is not supported"); - # the Anthropic SDK rewrites it to anyOf, so do the same. - result["anyOf"] = result.get("anyOf", []) + [ - AnthropicConfig.filter_anthropic_output_schema(item) for item in value - ] - else: - result[key] = value - - # Anthropic requires additionalProperties=false for object schemas - # See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs - if is_object_schema(result): - result["additionalProperties"] = False - - return result + def filter_anthropic_output_schema(schema: Mapping[str, object]) -> dict[str, object]: + """Shape ``schema`` for Anthropic's ``output_format`` API. See ``shape_claude_output_schema``.""" + return shape_claude_output_schema(schema) def get_json_schema_from_pydantic_object(self, response_format: type[BaseModel] | dict | None) -> dict | None: return type_to_response_format_param( @@ -1671,7 +1503,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): schema_has_unresolvable_ref(json_schema) or schema_has_open_object(json_schema) or schema_has_lookaround_pattern(json_schema) - or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES diff --git a/litellm/llms/anthropic/output_schema.py b/litellm/llms/anthropic/output_schema.py new file mode 100644 index 00000000000..d977c691d1a --- /dev/null +++ b/litellm/llms/anthropic/output_schema.py @@ -0,0 +1,217 @@ +""" +Shape a JSON schema into what Claude's structured-output grammar accepts, for +Anthropic ``output_format`` and Bedrock Converse and Invoke alike. Mirrors the +Anthropic SDK transform: +https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works +""" + +import json +from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.prompt_templates.common_utils import schema_fragment_ref + +# anthropic SDK ``lib/_parse/_transform.py`` SupportedStringFormats, checked 2026-10-13 +NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( + {"date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid"} +) + +_CONSTRAINT_NOTES: Final[Mapping[str, str]] = MappingProxyType( + { + "minItems": "minimum number of items: {}", + "maxItems": "maximum number of items: {}", + "uniqueItems": "all array items must be unique", + "contains": "array must contain an item matching: {}", + "minContains": "minimum number of matching items: {}", + "maxContains": "maximum number of matching items: {}", + "prefixItems": "leading items must match, in order: {}", + "minimum": "minimum value: {}", + "maximum": "maximum value: {}", + "exclusiveMinimum": "exclusive minimum value: {}", + "exclusiveMaximum": "exclusive maximum value: {}", + "multipleOf": "must be a multiple of {}", + "minLength": "minimum length: {}", + "maxLength": "maximum length: {}", + "minProperties": "minimum number of properties: {}", + "maxProperties": "maximum number of properties: {}", + "patternProperties": "properties whose names match each pattern must satisfy: {}", + "propertyNames": "property names must satisfy: {}", + "dependentRequired": "dependent required properties: {}", + "dependentSchemas": "dependent schemas: {}", + "unevaluatedProperties": "unevaluated properties must satisfy: {}", + "if": "conditional (if): {}", + "then": "conditional (then): {}", + "else": "conditional (else): {}", + "not": "must not match: {}", + } +) + +_SCHEMA_MAP_KEYS: Final = frozenset({"properties", "$defs", "definitions"}) +_COMPOSITION_KEYS: Final = frozenset({"anyOf", "oneOf", "allOf", "$ref"}) + +_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[object], bool]]] = MappingProxyType( + { + "null": lambda v: v is None, + "boolean": lambda v: isinstance(v, bool), + "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), + "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "string": lambda v: isinstance(v, str), + "array": lambda v: isinstance(v, list), + "object": lambda v: isinstance(v, dict), + } +) + +_SCHEMA: Final = TypeAdapter(Mapping[str, object]) +_ITEMS: Final = TypeAdapter(list[object]) +_TYPE_NAMES: Final = TypeAdapter(list[str]) + + +def _as_schema(value: object) -> Mapping[str, object] | None: + try: + return _SCHEMA.validate_python(value) + except ValidationError: + return None + + +def _as_items(value: object) -> list[object] | None: + try: + return _ITEMS.validate_python(value) + except ValidationError: + return None + + +def _declared_types(value: object) -> frozenset[str]: + if isinstance(value, str): + return frozenset((value,)) + try: + return frozenset(_TYPE_NAMES.validate_python(value)) + except ValidationError: + return frozenset() + + +def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool: + """Whether ``schema``'s ``enum`` cannot match its declared ``type``.""" + enum_values: Final = _as_items(schema.get("enum")) + declared_type: Final = schema.get("type") + if enum_values is None or declared_type is None: + return False + if isinstance(declared_type, str): + check: Final = _ENUM_TYPE_CHECKS.get(declared_type) + return check is not None and not all(check(value) for value in enum_values) + return _as_items(declared_type) is not None + + +def _note_value(value: object) -> object: + return value if value is None or isinstance(value, (str, int, float)) else json.dumps(value) + + +def _keeps_min_items(schema: Mapping[str, object]) -> bool: + """Claude's grammar enforces ``minItems`` of 0 or 1 and rejects anything larger.""" + value: Final = schema.get("minItems") + return isinstance(value, int) and not isinstance(value, bool) and value in (0, 1) + + +def _dropped_constraints(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset(_CONSTRAINT_NOTES) - ({"minItems"} if _keeps_min_items(schema) else frozenset()) + + +def _constraint_notes(schema: Mapping[str, object]) -> Iterator[str]: + dropped: Final = _dropped_constraints(schema) + for key, label in _CONSTRAINT_NOTES.items(): + if key in dropped and schema.get(key, False) is not False: + yield label.format(_note_value(schema[key])) + + +def _may_be_string(declared_type: object) -> bool: + return declared_type is None or "string" in _declared_types(declared_type) + + +def _unsupported_format(schema: Mapping[str, object], declared_type: object) -> str | None: + """Claude rejects string formats outside its list, while numeric formats such as ``int64`` pass through.""" + value: Final = schema.get("format") + if _may_be_string(declared_type) and isinstance(value, str) and value not in NATIVE_OUTPUT_STRING_FORMATS: + return value + return None + + +def _description_field(existing: object, notes: tuple[str, ...]) -> Mapping[str, object]: + if not notes: + return {} + note: Final = "Note: " + ", ".join(notes) + "." + if isinstance(existing, str) and existing: + return {"description": existing + " " + note} + return {"description": note} + + +def _object_fields(declared_type: object, schema: Mapping[str, object]) -> Mapping[str, object]: + """Claude 400s on any object node not closed with ``additionalProperties: false`` or missing ``type``.""" + if "object" in _declared_types(declared_type): + return {"additionalProperties": False} + if ( + declared_type is None + and "enum" not in schema + and _as_schema(schema.get("properties")) is not None + and not _COMPOSITION_KEYS & schema.keys() + ): + return {"type": "object", "additionalProperties": False} + return {} + + +def _shape_if_schema(value: object) -> object: + nested: Final = _as_schema(value) + return value if nested is None else shape_claude_output_schema(nested) + + +def _shape_each(value: object) -> object: + items: Final = _as_items(value) + return value if items is None else [_shape_if_schema(item) for item in items] + + +def _shape_nested(key: str, value: object) -> object: + if key in _SCHEMA_MAP_KEYS: + members: Final = _as_schema(value) + return value if members is None else {name: _shape_if_schema(member) for name, member in members.items()} + if key == "items": + return _shape_if_schema(value) + if key == "allOf": + return _shape_each(value) + if key == "$ref" and isinstance(value, str): + return schema_fragment_ref(value) + return value + + +def _any_of_field(schema: Mapping[str, object]) -> Mapping[str, object]: + variants: Final = [*(_as_items(schema.get("anyOf")) or ()), *(_as_items(schema.get("oneOf")) or ())] + return {"anyOf": [_shape_if_schema(variant) for variant in variants]} if variants else {} + + +def shape_claude_output_schema(schema: Mapping[str, object]) -> dict[str, object]: + """Return a new schema that Claude's structured-output API accepts. Idempotent.""" + drops_type: Final = _enum_conflicts_with_declared_type(schema) + declared_type: Final = None if drops_type else schema.get("type") + object_fields: Final = {} if drops_type else _object_fields(declared_type, schema) + unsupported_format: Final = _unsupported_format(schema, declared_type) + notes: Final = ( + *_constraint_notes(schema), + *(() if unsupported_format is None else (f"format: {unsupported_format}",)), + ) + dropped: Final = frozenset( + (*_dropped_constraints(schema), "anyOf", "oneOf") + + (("type",) if drops_type else ()) + + (("additionalProperties",) if object_fields else ()) + + (("description",) if notes else ()) + + (("format",) if unsupported_format is not None else ()) + ) + kept: Final = {key: _shape_nested(key, value) for key, value in schema.items() if key not in dropped} + return {**_description_field(schema.get("description"), notes), **kept, **_any_of_field(schema), **object_fields} + + +def shape_claude_output_format(output_format: Mapping[str, object]) -> dict[str, object]: + """Shape the ``schema`` inside an Anthropic ``output_format`` block, leaving the rest untouched.""" + schema: Final = _as_schema(output_format.get("schema")) + if schema is None: + return dict(output_format) + return {**output_format, "schema": shape_claude_output_schema(schema)} diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 443e722168d..5980e06c2e7 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -68,6 +68,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_native_output_is_opt_in, bedrock_supports_tool_search, deployment_model_info, + json_object_or_none, requests_output_config_format, ) from litellm.llms.bedrock.request_metadata import ( @@ -970,17 +971,20 @@ class AmazonConverseConfig(BaseConfig): def _native_output_schema( self, model: str, - json_schema: Mapping[str, object], + json_schema: object, thinking_requested: bool, model_info: Mapping[str, object] | None, - ) -> Mapping[str, object] | None: + ) -> object | None: if not self._supports_native_structured_outputs( model, self.custom_llm_provider, thinking_requested, model_info ): return None + schema: Final = json_object_or_none(json_schema) + if schema is None: + return json_schema if BedrockModelInfo.get_base_model(model).startswith("anthropic"): - return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse") - return json_schema + return bedrock_cleaned_output_schema(model, schema, self.custom_llm_provider or "bedrock_converse") + return schema def _output_config_for_anthropic_format( self, model: str, output_config_format: Mapping[str, object], model_info: Mapping[str, object] | None @@ -999,7 +1003,7 @@ class AmazonConverseConfig(BaseConfig): @staticmethod def _create_output_config_for_response_format( - json_schema: dict | None = None, + json_schema: object | None = None, name: str | None = None, description: str | None = None, ) -> "OutputConfigBlock": @@ -1022,9 +1026,12 @@ class AmazonConverseConfig(BaseConfig): } } """ - if json_schema is not None: - json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema) - schema_str: Final = json.dumps(json_schema) if json_schema is not None else "{}" + closed_schema: Final = ( + AmazonConverseConfig._add_additional_properties_to_schema(json_schema) + if isinstance(json_schema, dict) + else json_schema + ) + schema_str: Final = json.dumps(closed_schema) if closed_schema is not None else "{}" json_schema_def: Final[JsonSchemaDefinition] = {"schema": schema_str} if name is not None: json_schema_def["name"] = name @@ -1322,7 +1329,7 @@ class AmazonConverseConfig(BaseConfig): # Requires an explicit schema — json_object with no schema falls through # to the tool-call path below. output_config: Final = self._create_output_config_for_response_format( - json_schema=dict(native_schema), + json_schema=dict(native_schema) if isinstance(native_schema, Mapping) else native_schema, name=name, description=description, ) diff --git a/tests/e2e/llm_translation/structured_output.py b/tests/e2e/llm_translation/structured_output.py index b20fe769eaa..99cb222ada6 100644 --- a/tests/e2e/llm_translation/structured_output.py +++ b/tests/e2e/llm_translation/structured_output.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Final -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter SENTIMENT_PROMPT: Final = "Classify the sentiment of this review: 'The battery died after two days.'" SENTIMENT_LABELS: Final = frozenset({"positive", "negative", "neutral"}) @@ -15,10 +15,36 @@ SENTIMENT_OUTPUT_FORMAT: Final[dict[str, object]] = { "additionalProperties": False, }, } +CONSTRAINED_SENTIMENT_OUTPUT_FORMAT: Final[dict[str, object]] = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "sentiment": {"type": "string", "enum": sorted(SENTIMENT_LABELS)}, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "reasons": {"type": "array", "items": {"type": "string"}, "maxItems": 3}, + }, + "required": ["sentiment", "confidence", "reasons"], + "additionalProperties": False, + }, +} _SENTIMENT_JSON: Final = TypeAdapter(dict[str, str]) +class _ConstrainedSentiment(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + sentiment: str + confidence: float = Field(ge=0, le=1) + reasons: tuple[str, ...] = Field(max_length=3) + + def assert_sentiment_json(text: str) -> None: parsed = _SENTIMENT_JSON.validate_json(text) assert set(parsed) == {"sentiment"}, f"output_format schema not enforced, extra or missing keys: {parsed}" assert parsed["sentiment"] in SENTIMENT_LABELS, f"sentiment outside the schema enum: {parsed}" + + +def assert_constrained_sentiment_json(text: str) -> None: + parsed = _ConstrainedSentiment.model_validate_json(text) + assert parsed.sentiment in SENTIMENT_LABELS, f"sentiment outside the schema enum: {parsed}" diff --git a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py index 31daa9bba32..d28d5c38fe3 100644 --- a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py +++ b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py @@ -10,7 +10,13 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from sdk_clients import NO_PROXY_CACHE, SdkClients -from structured_output import SENTIMENT_OUTPUT_FORMAT, SENTIMENT_PROMPT, assert_sentiment_json +from structured_output import ( + CONSTRAINED_SENTIMENT_OUTPUT_FORMAT, + SENTIMENT_OUTPUT_FORMAT, + SENTIMENT_PROMPT, + assert_constrained_sentiment_json, + assert_sentiment_json, +) pytestmark = pytest.mark.e2e @@ -136,6 +142,32 @@ class TestBedrockMessages: assert len(texts) == len(message.content), f"structured output came back as non-text blocks: {message!r}" assert_sentiment_json("".join(texts)) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(CONVERSE_CLAUDE_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) + def test_converse_output_format_accepts_constraint_keywords( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, CONVERSE_CLAUDE_BACKEND) + client = sdk.anthropic(resources.key()) + + message = client.messages.create( + model=model, + max_tokens=256, + messages=[{"role": "user", "content": SENTIMENT_PROMPT}], + extra_body={**NO_PROXY_CACHE, "output_format": CONSTRAINED_SENTIMENT_OUTPUT_FORMAT}, + ) + texts = tuple(block.text for block in message.content if isinstance(block, TextBlock)) + assert len(texts) == len(message.content), f"structured output came back as non-text blocks: {message!r}" + assert_constrained_sentiment_json("".join(texts)) + @pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works") @meta( Subject( diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 5803d1dee44..3f31b25b8ac 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -88,6 +88,15 @@ _LIMITS_SCHEMA: Final[dict[str, JsonValue]] = { "required": ["tier", "summary", "tags", "owner", "backup"], "$defs": {"Person": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}, } +_UNLISTED_FORMAT_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "id": {"type": "string", "format": "int64"}, + "meta": {"properties": {"a": {"type": "string"}}, "required": ["a"]}, + }, + "required": ["id", "meta"], + "additionalProperties": False, +} _OPEN_COUNTS: Final[dict[str, JsonValue]] = {"type": "object", "additionalProperties": {"type": "integer"}} _DICT_SCHEMA: Final[dict[str, JsonValue]] = { "type": "object", @@ -754,6 +763,35 @@ def test_native_schema_reaches_bedrock_with_limits_in_descriptions_and_every_obj assert _JSON_TOOL not in _tools(body, route == "invoke"), body +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_unlisted_string_format_reaches_bedrock_as_a_description_note_with_the_untyped_node_closed( + gateway: Gateway, route: str +) -> None: + """Bedrock Claude rejects a string ``format`` outside Claude's list and a node with no ``type``, so the + proxy notes the format in the description and types the node, instead of dropping to the json tool.""" + marker, marker_hex = _marker() + with wire_server(_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _SONNET) + body: Final = _chat_body(model, marker, response_format=_response_format(_UNLISTED_FORMAT_SCHEMA)) + assert _content_json(_chat_message(gateway.request("POST", "/v1/chat/completions", body))) == _payload( + marker_hex + ) + _, sent = _one_call(wire, marker_hex) + schema: Final = _native_schema(sent, route == "invoke") + assert schema is not None, sent + assert _bedrock_schema_fault(schema) is None, schema + assert object_value(schema["properties"]) == { + "id": {"type": "string", "description": "Note: format: int64."}, + "meta": { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "additionalProperties": False, + }, + }, schema + assert _JSON_TOOL not in _tools(sent, route == "invoke"), sent + + @pytest.mark.parametrize("route", ["converse", "invoke"]) def test_open_object_schema_takes_the_json_tool_on_a_native_model_and_keeps_the_dict_open( gateway: Gateway, route: str diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 4e31122440d..e1d98df89f5 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -1689,6 +1689,53 @@ def test_vertex_chat_native_schema_keeps_min_items_one_and_closes_a_nullable_obj assert "tools" not in sent.body, sent.body +def test_anthropic_chat_native_schema_notes_an_unlisted_string_format_and_types_an_untyped_node( + gateway: Gateway, +) -> None: + """Anthropic 400s on a string ``format`` outside Claude's list ("format 'int64' is not supported") and on + a node with properties but no ``type`` ("Schema type is missing"), so the proxy rewrites both.""" + marker, marker_hex = _marker("native") + schema: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": { + "id": {"type": "string", "format": "int64"}, + "meta": {"properties": {"a": {"type": "string"}}, "required": ["a"]}, + }, + "required": ["id", "meta"], + "additionalProperties": False, + } + expected: Final[dict[str, JsonValue]] = { + **schema, + "properties": { + "id": {"description": "Note: format: int64.", "type": "string"}, + "meta": { + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "type": "object", + "additionalProperties": False, + }, + }, + } + response_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "record", "schema": schema}, + } + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, "anthropic", _NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=response_format) + ) + assert _chat_content(response) == _payload(marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert _native_format(sent.body) == {"type": "json_schema", "schema": expected}, sent.body + assert "tools" not in sent.body, sent.body + + +def _native_format(body: Mapping[str, JsonValue]) -> JsonValue: + output_format: Final = body.get("output_format") + return output_format if output_format is not None else object_value(body["output_config"])["format"] + + @pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) def test_vertex_messages_folds_output_format_into_output_config(gateway: Gateway, stream: bool) -> None: marker, marker_hex = _marker("native") diff --git a/tests/unit/llms/anthropic/test_anthropic_output_schema.py b/tests/unit/llms/anthropic/test_anthropic_output_schema.py new file mode 100644 index 00000000000..4133e165e9e --- /dev/null +++ b/tests/unit/llms/anthropic/test_anthropic_output_schema.py @@ -0,0 +1,106 @@ +"""Tests for the shared Claude structured-output schema shaper.""" + +import copy + +import pytest + +from litellm.llms.anthropic.output_schema import ( + shape_claude_output_format, + shape_claude_output_schema, +) + +_MESSY_SCHEMA = { + "type": "object", + "description": "A record", + "properties": { + "tags": {"type": "array", "items": {"type": "string"}, "maxItems": 3, "uniqueItems": True}, + "score": {"type": "number", "minimum": 0, "exclusiveMaximum": 1}, + "id": {"type": "string", "format": "int64", "minLength": 1}, + "meta": {"type": ["object", "null"], "properties": {"a": {"type": "string"}}}, + "kind": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, + "labels": {"type": "object", "additionalProperties": {"type": "string"}}, + "nested": {"properties": {"b": {"type": "boolean"}}}, + }, + "required": ["tags", "score", "id"], + "additionalProperties": True, +} + + +def test_shaping_is_idempotent(): + once = shape_claude_output_schema(_MESSY_SCHEMA) + + assert shape_claude_output_schema(once) == once + + +def test_shaping_does_not_mutate_the_input(): + before = copy.deepcopy(_MESSY_SCHEMA) + + shape_claude_output_schema(_MESSY_SCHEMA) + + assert _MESSY_SCHEMA == before + + +# Claude-supported formats per the anthropic SDK ``lib/_parse/_transform.py`` SupportedStringFormats, checked 2026-10-13 +@pytest.mark.parametrize("fmt", ["date-time", "date", "email", "uri", "uuid"]) +def test_supported_string_format_is_kept(fmt: str): + assert shape_claude_output_schema({"type": "string", "format": fmt}) == {"type": "string", "format": fmt} + + +_BOUNDED = {"type": "number", "minimum": 0} +_BOUNDED_SHAPED = {"type": "number", "description": "Note: minimum value: 0."} + + +@pytest.mark.parametrize( + ("schema", "expected"), + [ + ({"$defs": {"X": _BOUNDED}}, {"$defs": {"X": _BOUNDED_SHAPED}}), + ({"definitions": {"X": _BOUNDED}}, {"definitions": {"X": _BOUNDED_SHAPED}}), + ({"allOf": [_BOUNDED]}, {"allOf": [_BOUNDED_SHAPED]}), + ], + ids=["$defs", "definitions", "allOf"], +) +def test_shapes_schemas_nested_under_definitions_and_all_of(schema: dict[str, object], expected: dict[str, object]): + assert shape_claude_output_schema(schema) == expected + + +@pytest.mark.parametrize("composition", ["anyOf", "oneOf", "allOf", "$ref"]) +def test_untyped_node_with_properties_and_composition_is_not_forced_to_object(composition: str): + value = "#/$defs/X" if composition == "$ref" else [{"type": "object", "properties": {}}] + result = shape_claude_output_schema({"properties": {"a": {"type": "string"}}, composition: value}) + + assert "type" not in result + assert "additionalProperties" not in result + + +def test_output_format_block_shapes_only_its_schema(): + block = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"n": {"type": "integer", "minimum": 1}}}, + } + + result = shape_claude_output_format(block) + + assert result == { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"n": {"type": "integer", "description": "Note: minimum value: 1."}}, + "additionalProperties": False, + }, + } + assert block["schema"]["properties"]["n"] == {"type": "integer", "minimum": 1} + + +def test_output_format_block_without_schema_is_returned_unchanged(): + assert shape_claude_output_format({"type": "json_schema"}) == {"type": "json_schema"} + + +def test_nullable_object_enum_keeps_its_type_dropped_instead_of_forcing_object() -> None: + """``enum: [null]`` on an ``["object", "null"]`` node conflicts with the declared type, so the type + is dropped. Re-adding ``type: object`` would make the node impossible to satisfy.""" + shaped = shape_claude_output_schema( + {"type": ["object", "null"], "properties": {"a": {"type": "string"}}, "enum": [None]} + ) + + assert shaped == {"properties": {"a": {"type": "string"}}, "enum": [None]} + assert shape_claude_output_schema(shaped) == shaped diff --git a/tests/unit/llms/anthropic/test_anthropic_schema_filter.py b/tests/unit/llms/anthropic/test_anthropic_schema_filter.py index 407a52c183e..2ba8285817b 100644 --- a/tests/unit/llms/anthropic/test_anthropic_schema_filter.py +++ b/tests/unit/llms/anthropic/test_anthropic_schema_filter.py @@ -526,3 +526,46 @@ class TestFilterAnthropicOutputSchema: result = AnthropicConfig.filter_anthropic_output_schema(schema) assert "type" not in result["properties"]["rows"]["items"]["properties"]["c"] + + def test_closes_nullable_object(self): + """``type: ["object", "null"]`` is still an object and must be closed too, + otherwise the API answers "'additionalProperties' must be explicitly set + to false".""" + schema = { + "type": "object", + "properties": {"meta": {"type": ["object", "null"], "properties": {"a": {"type": "string"}}}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["meta"]["additionalProperties"] is False + assert result["properties"]["meta"]["type"] == ["object", "null"] + + def test_closes_untyped_node_that_declares_properties(self): + """A node with ``properties`` but no ``type`` is an object by structure. + The API rejects it with "Schema type is missing", so it is typed and closed.""" + schema = { + "type": "object", + "properties": {"meta": {"properties": {"a": {"type": "string"}}, "required": ["a"]}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["meta"] == { + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "type": "object", + "additionalProperties": False, + } + + def test_moves_unsupported_string_format_into_description(self): + """A ``format`` outside the supported set 400s ("format 'int64' is not + supported"), so it is dropped and noted like any other constraint.""" + schema = { + "type": "object", + "properties": {"id": {"type": "string", "format": "int64", "description": "row id"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["id"] == {"type": "string", "description": "row id Note: format: int64."} diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index d82ba192b1f..98d444e2ab5 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5157,9 +5157,8 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( (ROOT_REF_SCHEMA, "$ref"), (OPEN_OBJECT_SCHEMA, "additionalProperties"), (LOOKAROUND_PATTERN_SCHEMA, "pattern"), - (UNKNOWN_FORMAT_SCHEMA, "format"), ], - ids=["recursive", "root-ref", "open-object", "lookaround-pattern", "unlisted-format"], + ids=["recursive", "root-ref", "open-object", "lookaround-pattern"], ) def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( set_native_structured_output, schema, kept_keyword, native_key @@ -5175,6 +5174,22 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil assert kept_keyword in _schema_keys(tool_spec["inputSchema"]["json"]) +@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) +def test_unlisted_string_format_moves_into_the_description_and_keeps_native_output( + set_native_structured_output, native_key +): + """Claude's grammar rejects a string ``format`` outside its list ("format + 'x-litellm-unlisted-format' is not supported"), so the shaper drops it and + notes it, the way it does for other constraints, instead of leaving native output.""" + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, native_key) + + request: Final = _converse_request_for_schema(model, UNKNOWN_FORMAT_SCHEMA) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert sent["properties"]["blob"] == {"type": "string", "description": "Note: format: x-litellm-unlisted-format."} + + def _shared_defs_schema(defs_key: str) -> Mapping[str, object]: party_ref: Final = {"$ref": f"#/{defs_key}/Party"} return { @@ -5784,6 +5799,81 @@ def test_output_config_applies_additional_properties(): assert parsed["properties"]["nested"]["additionalProperties"] is False +_SCHEMA_CLAUDE_REJECTS = { + "type": "object", + "properties": { + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "tags": {"type": "array", "items": {"type": "string"}, "maxItems": 3}, + "meta": {"type": ["object", "null"], "properties": {"a": {"type": "string"}}}, + }, + "required": ["score"], +} + + +def _converse_schema_json(model: str, schema: object) -> str: + response_format = {"type": "json_schema", "json_schema": {"name": "Score", "schema": schema}} + result = AmazonConverseConfig()._translate_response_format_param( + value=response_format, + model=model, + optional_params={}, + non_default_params={"response_format": response_format}, + is_thinking_enabled=False, + ) + return result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] + + +def _converse_schema_for(model: str) -> dict: + return json.loads(_converse_schema_json(model, _SCHEMA_CLAUDE_REJECTS)) + + +def test_claude_response_format_schema_shaped_like_anthropic_path(local_model_cost_map): + """Bedrock relays a Claude schema to Claude's grammar compiler, which 400s on + keywords it cannot enforce ("For 'array' type, property 'maxItems' is not + supported") and on any object that is not closed, so Converse shapes it like + the direct Anthropic path.""" + assert _converse_schema_for("us.anthropic.claude-sonnet-4-6") == { + "type": "object", + "properties": { + "score": {"type": "number", "description": "Note: minimum value: 0, maximum value: 1."}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Note: maximum number of items: 3."}, + "meta": { + "type": ["object", "null"], + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + }, + }, + "required": ["score"], + "additionalProperties": False, + } + + +def test_non_claude_response_format_schema_keeps_constraints(local_model_cost_map): + """Non-Claude Bedrock models enforce these keywords, so only the missing + ``additionalProperties`` is filled in.""" + assert _converse_schema_for("qwen.qwen3-32b-v1:0") == { + "type": "object", + "properties": { + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "tags": {"type": "array", "items": {"type": "string"}, "maxItems": 3}, + "meta": { + "type": ["object", "null"], + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + }, + }, + "required": ["score"], + "additionalProperties": False, + } + + +@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-4-6", "qwen.qwen3-32b-v1:0"]) +@pytest.mark.parametrize("schema", [True, "not a schema"]) +def test_non_object_response_format_schema_is_forwarded_unchanged(local_model_cost_map, model: str, schema: object): + """A schema that is not an object reaches Bedrock as sent, so the caller gets + Bedrock's 400 instead of a 500 from the proxy.""" + assert json.loads(_converse_schema_json(model, schema)) == schema + + _TOOL_PARAM = [ { "type": "function", diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 0bd480dcdbc..cb559c05bc2 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -3376,7 +3376,7 @@ def test_bedrock_messages_legacy_output_format_wins_over_output_config_format(lo } newer_format = { "type": "json_schema", - "schema": {"type": "object", "properties": {"newer_field": {"type": "string"}}}, + "schema": {"type": "object", "properties": {"newer_field": {"type": "string"}}, "additionalProperties": False}, } result = cfg.transform_anthropic_messages_request( diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index dd89d9c46b6..c4ab1ca4145 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -689,6 +689,75 @@ def test_apply_structured_output_prefers_legacy_output_format(local_model_cost_m assert "output_format" not in body +_SCHEMA_WITH_REJECTED_KEYWORDS = { + "type": "object", + "properties": { + "tags": {"type": "array", "items": {"type": "string"}, "maxItems": 3}, + "meta": {"type": ["object", "null"], "properties": {"a": {"type": "string"}}}, + }, + "required": ["tags"], +} + + +def test_apply_structured_output_shapes_schema_for_native_models(local_model_cost_map): + """Bedrock relays the schema to Claude's grammar compiler, which 400s on + keywords it cannot enforce and on open objects, so the Invoke path shapes + the schema like the direct Anthropic path before forwarding it.""" + from litellm.llms.bedrock.common_utils import apply_bedrock_invoke_structured_output + + body = { + "messages": [{"role": "user", "content": "hi"}], + "output_format": {"type": "json_schema", "schema": _SCHEMA_WITH_REJECTED_KEYWORDS}, + } + + apply_bedrock_invoke_structured_output( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + request_body=body, + ) + + assert body["output_config"]["format"] == { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Note: maximum number of items: 3.", + }, + "meta": { + "type": ["object", "null"], + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + }, + }, + "required": ["tags"], + "additionalProperties": False, + }, + } + + +def test_apply_structured_output_inlines_raw_schema_for_other_models(local_model_cost_map): + """The prompt-engineered fallback is text the model reads, not a grammar, so + the schema is inlined untouched.""" + import json + + from litellm.llms.bedrock.common_utils import apply_bedrock_invoke_structured_output + + body = { + "messages": [{"role": "user", "content": "hi"}], + "output_format": {"type": "json_schema", "schema": _SCHEMA_WITH_REJECTED_KEYWORDS}, + } + + apply_bedrock_invoke_structured_output( + model="anthropic.claude-3-haiku-20240307-v1:0", + request_body=body, + ) + + assert "output_config" not in body + assert json.loads(body["messages"][-1]["content"][-1]["text"]) == _SCHEMA_WITH_REJECTED_KEYWORDS + + def test_sign_aws_request_assumes_role_with_external_id(monkeypatch): """A trust policy requiring sts:ExternalId must be satisfied when signing batch API requests.""" import datetime