scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(model_map): flag native structured outputs on Anthropic-direct claude-sonnet-5 and claude-haiku-4-5 (#35930)

Browse source 
* fix(model_map): flag native structured outputs on Anthropic-direct claude-sonnet-5 and claude-haiku-4-5

The Bedrock twins of both models already carry
supports_native_structured_output, but the Anthropic-direct entries do not,
so response_format requests to anthropic/claude-sonnet-5 and
anthropic/claude-haiku-4-5 fall back to the json_tool_call emulation and
inherit its nested-envelope failure modes (#8898) despite the API supporting
output_format natively.

Verified live against the Anthropic API on 2026-08-05: both models accept
output_format (structured outputs beta header) and return exact schema
instances, including a large nested production schema validated with
pydantic. Same two lines applied to the bundled backup map.

* fix(model_map): cover the versioned claude-haiku-4-5-20251001 alias

Exact-match capability lookup of anthropic/claude-haiku-4-5-20251001
resolved the versioned entry, which lacked the flag, so response_format
for that identifier still took the tool-emulation path. Flag it in both
the root and bundled maps, matching its unversioned alias.

* fix(anthropic): bound $defs inlining in output_format with the shared schema-bomb budget

map_response_format_to_anthropic_output_format called unpack_defs with
no max_inlined_bytes, so an authenticated caller could send a compact
schema whose repeated $refs expand without bound before reaching the
provider. Reuse the existing 10MB inlining budget (renamed from
_LEGACY_DEFS_MAX_INLINED_BYTES to DEFS_MAX_INLINED_BYTES now that two
call sites share it); overflow raises ValueError instead of
materialising the expansion.

Regression tests: a compact schema bomb is rejected, a normal $defs
schema still resolves; the bomb test fails when the bound is removed.

* chore: retrigger CI (benchmarks job flaked on a PyPI download timeout)

---------

Co-authored-by: Anmol Jaiswal <anmolg1997@users.noreply.github.com>
This commit is contained in:
Anmol Jaiswal 2026-08-16 00:21:35 +05:30 • committed by GitHub
parent fe9451c6cd
commit 0059b497f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 54 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1004,7 +1004,7 @@ def _has_legacy_defs(schema: object) -> bool:
return "definitions" in schema or (isinstance(components, dict) and isinstance(components.get("schemas"), dict))
# Schema-bomb budget for ``unpack_legacy_defs``: cap the cumulative JSON-byte
# Schema-bomb budget for ``$ref`` inlining: cap the cumulative JSON-byte
# size of every inlined target. A byte cap is the universal measure of
# expansion -- it simultaneously bounds ref-count fan-out, node-count
# amplification, and scalar-byte amplification (large ``description`` /
@ -1012,14 +1012,14 @@ def _has_legacy_defs(schema: object) -> bool:
# inline well under 1MB; 10MB sits two orders of magnitude above that, well
# below memory-pressure territory, and rejects request-supplied bombs before
# the proxy materialises them.
_LEGACY_DEFS_MAX_INLINED_BYTES: Final = 10_000_000
DEFS_MAX_INLINED_BYTES: Final = 10_000_000
def unpack_legacy_defs(
schema: dict,
*,
copy: bool = False,
max_inlined_bytes: int = _LEGACY_DEFS_MAX_INLINED_BYTES,
max_inlined_bytes: int = DEFS_MAX_INLINED_BYTES,
) -> dict:
"""Inline ``$ref``s backed by draft-04 ``definitions`` / OpenAPI
``components.schemas``. ``$defs`` is left untouched.

3
litellm/llms/anthropic/chat/transformation.py
View file

@ -1267,13 +1267,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs)
unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema: Final = self.filter_anthropic_output_schema(json_schema)

3
litellm/model_prices_and_context_window_backup.json
View file

@ -12011,6 +12011,7 @@
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
@ -12033,6 +12034,7 @@
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
@ -12254,6 +12256,7 @@
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

3
model_prices_and_context_window.json
View file

@ -12011,6 +12011,7 @@
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
@ -12033,6 +12034,7 @@
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_computer_use": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
@ -12254,6 +12256,7 @@
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

59
tests/test_litellm/llms/anthropic/test_anthropic_structured_output.py
View file

@ -46,9 +46,7 @@ class TestAnthropicStructuredOutput:
"json_schema": json_schema["json_schema"],
}
output_format = config.map_response_format_to_anthropic_output_format(
response_format
)
output_format = config.map_response_format_to_anthropic_output_format(response_format)
# Verify that maxItems is filtered out for Anthropic
assert output_format is not None
@ -82,9 +80,7 @@ class TestAnthropicStructuredOutput:
"json_schema": json_schema["json_schema"],
}
output_format = config.map_response_format_to_anthropic_output_format(
response_format
)
output_format = config.map_response_format_to_anthropic_output_format(response_format)
assert output_format is not None
transformed_schema = output_format["schema"]
@ -112,9 +108,7 @@ class TestAnthropicStructuredOutput:
"json_schema": json_schema["json_schema"],
}
output_format = config.map_response_format_to_anthropic_output_format(
response_format
)
output_format = config.map_response_format_to_anthropic_output_format(response_format)
assert output_format is not None
transformed_schema = output_format["schema"]
@ -125,10 +119,7 @@ class TestAnthropicStructuredOutput:
# Nested maxItems should also be removed
if "$defs" in transformed_schema:
nested_item_schema = transformed_schema["$defs"].get("NestedItem", {})
if (
"properties" in nested_item_schema
and "tags" in nested_item_schema["properties"]
):
if "properties" in nested_item_schema and "tags" in nested_item_schema["properties"]:
assert "maxItems" not in nested_item_schema["properties"]["tags"]
def test_other_constraints_preserved(self):
@ -153,9 +144,7 @@ class TestAnthropicStructuredOutput:
"json_schema": json_schema["json_schema"],
}
output_format = config.map_response_format_to_anthropic_output_format(
response_format
)
output_format = config.map_response_format_to_anthropic_output_format(response_format)
assert output_format is not None
transformed_schema = output_format["schema"]
@ -177,3 +166,41 @@ class TestAnthropicStructuredOutput:
assert "description" in age_schema
assert "minimum value: 0" in age_schema["description"]
assert "maximum value: 150" in age_schema["description"]
class TestAnthropicOutputFormatSchemaBudget:
"""The $defs inlining in map_response_format_to_anthropic_output_format is byte-bounded."""
@staticmethod
def _response_format(schema: dict) -> dict:
return {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}
def test_schema_bomb_rejected(self):
"""A compact request whose $defs expand past the byte budget raises instead of materialising."""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
big = {"type": "string", "description": "x" * 200_000}
schema = {
"type": "object",
"$defs": {"Big": big},
"properties": {f"p{i}": {"$ref": "#/$defs/Big"} for i in range(60)},
}
with pytest.raises(ValueError, match="budget"):
AnthropicConfig().map_response_format_to_anthropic_output_format(self._response_format(schema))
def test_normal_defs_still_resolve(self):
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
schema = {
"type": "object",
"$defs": {"Item": {"type": "string", "description": "an item"}},
"properties": {"a": {"$ref": "#/$defs/Item"}, "b": {"$ref": "#/$defs/Item"}},
}
output_format = AnthropicConfig().map_response_format_to_anthropic_output_format(self._response_format(schema))
assert output_format is not None
resolved = output_format["schema"]["properties"]
assert resolved["a"]["type"] == "string"
assert resolved["b"]["type"] == "string"
assert "$ref" not in str(resolved)