fix(bedrock): merge root ref siblings, drop unused $defs, and use the shared inlining budget

A root $ref beside an empty properties block, the shape MCP tools get, now
merges into its target instead of replacing the target's properties, so a
strict Claude tool sends its real arguments again. Claude also stops getting
$defs entries no $ref reaches, which kept unused definitions out of its
input tokens before this branch. Inlining for other models and for legacy
definitions now uses the same 10 MB budget the native Anthropic route uses,
since 1 MB refused an 852 KB expansion that worked at the merge base, and
the 400 now names the bytes of definitions copied
This commit is contained in:
mateo-berri 2026-11-11 05:41:58 -08:00
parent 75da9c820e
commit 6fc409da1a
5 changed files with 177 additions and 13 deletions

View file

@ -1092,7 +1092,6 @@ def _legacy_defs(schema: object) -> Mapping[str, object] | None:
# below memory-pressure territory, and rejects request-supplied bombs before
# the proxy materialises them.
DEFS_MAX_INLINED_BYTES: Final = 10_000_000
BEDROCK_TOOLS_MAX_INLINED_BYTES: Final = 1_000_000
def unpack_legacy_defs(

View file

@ -65,7 +65,7 @@ from litellm.types.llms.vertex_ai import PartType as VertexPartType
from litellm.types.utils import GenericImageParsingChunk
from .common_utils import (
BEDROCK_TOOLS_MAX_INLINED_BYTES,
DEFS_MAX_INLINED_BYTES,
as_openai_image_part,
converse_redacted_content,
convert_content_list_to_str,
@ -81,7 +81,7 @@ from .common_utils import (
unpack_defs,
)
from .image_handling import convert_url_to_base64, is_pdf_media_type
from .json_schema_utils import schema_with_root_ref_resolved
from .json_schema_utils import schema_with_root_ref_resolved, schema_without_unreferenced_defs
def default_pt(messages):
@ -5949,11 +5949,11 @@ def _bedrock_tools_pt(
else:
description = name
remaining_inline_bytes = BEDROCK_TOOLS_MAX_INLINED_BYTES - inlined_bytes
remaining_inline_bytes = DEFS_MAX_INLINED_BYTES - inlined_bytes
try:
if resolves_schema_defs:
tool_inlined_bytes = inline_legacy_defs(parameters, remaining_inline_bytes)
parameters = dict(schema_with_root_ref_resolved(parameters))
parameters = dict(schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters)))
else:
defs = parameters.pop("$defs", {})
defs_copy = defs if isinstance(defs, dict) else {}
@ -5965,8 +5965,9 @@ def _bedrock_tools_pt(
except ValueError as e:
raise BedrockError(
status_code=400,
message=f"Inlining the $ref definitions of tool '{raw_name}' pushes the request's tool schemas past "
f"{BEDROCK_TOOLS_MAX_INLINED_BYTES:,} bytes. Reference shared $defs less often or flatten the schema.",
message=f"Inlining the $ref definitions of tool '{raw_name}' copies more than "
f"{DEFS_MAX_INLINED_BYTES:,} bytes of definitions across the request's tools. "
"Reference shared $defs less often or flatten the schema.",
) from e
inlined_bytes += tool_inlined_bytes
normalize_json_schema_custom_types_to_object(parameters)

View file

@ -5,6 +5,7 @@ from graphlib import CycleError, TopologicalSorter
from itertools import chain, islice
from types import MappingProxyType
from typing import Final
from urllib.parse import unquote
from pydantic import TypeAdapter
@ -13,6 +14,8 @@ _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not")
_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions"))
_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32
_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({})
_ROOT_DEFS_POINTER: Final = "#/$defs/"
_DYNAMIC_REF_KEYWORDS: Final = ("$dynamicRef", "$recursiveRef")
def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]:
@ -56,18 +59,76 @@ def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[s
return target if isinstance(target, dict) else None
def _required_names_in_order(schema: Mapping[str, object]) -> tuple[str, ...]:
return tuple(name for name in _list_at(schema, "required") if isinstance(name, str))
def _with_root_ref_inlined(schema: Mapping[str, object]) -> Mapping[str, object]:
ref: Final = schema.get("$ref")
target: Final = _resolve_local_schema_ref(schema, ref) if isinstance(ref, str) else None
if target is None:
return schema
return {**target, **{key: value for key, value in schema.items() if key != "$ref"}}
siblings: Final = {key: value for key, value in schema.items() if key != "$ref"}
required: Final = dict.fromkeys((*_required_names_in_order(target), *_required_names_in_order(schema)))
return {
**target,
**siblings,
"properties": {**_schema_properties(target), **_schema_properties(schema)},
"required": list(required),
}
def schema_with_root_ref_resolved(schema: Mapping[str, object]) -> Mapping[str, object]:
return reduce(lambda resolved, _hop: _with_root_ref_inlined(resolved), range(_MAX_SCHEMA_FLATTEN_DEPTH), schema)
def _referenced_def_names(ref: object) -> frozenset[str] | None:
if not isinstance(ref, str):
return frozenset()
pointer: Final = schema_fragment_ref(ref)
if pointer == "#" or (pointer.startswith("#/") and not pointer.startswith("#/$defs")):
return frozenset()
if not pointer.startswith(_ROOT_DEFS_POINTER):
return None
name: Final = pointer[len(_ROOT_DEFS_POINTER) :].split("/")[0]
return frozenset((name, unquote(name).replace("~1", "/").replace("~0", "~")))
def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]:
if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS):
return (None,)
return (_referenced_def_names(node["$ref"]),) if "$ref" in node else ()
def _schema_ref_def_names(schema: Mapping[str, object]) -> Iterator[frozenset[str] | None]:
return chain.from_iterable(_node_ref_def_names(node) for node in _schema_nodes(schema))
def _reached_def_names(body: Mapping[str, object], defs: Mapping[str, object]) -> frozenset[str] | None:
reached: frozenset[str] = frozenset() # rebind-ok: grows by the definitions the last round's refs name
frontier: tuple[Mapping[str, object], ...] = (body,) # rebind-ok: the schemas the last round reached
while frontier:
found = tuple(chain.from_iterable(_schema_ref_def_names(node) for node in frontier))
if None in found:
return None
new_names = frozenset(chain.from_iterable(names for names in found if names is not None)) - reached
reached = reached | new_names
frontier = tuple(definition for name in new_names if (definition := _schema_at(defs, name)) is not None)
return reached
def schema_without_unreferenced_defs(schema: Mapping[str, object]) -> Mapping[str, object]:
defs: Final = _schema_at(schema, "$defs")
if defs is None:
return schema
body: Final = {key: value for key, value in schema.items() if key != "$defs"}
reached: Final = _reached_def_names(body, defs)
if reached is None:
return schema
kept: Final = {name: definition for name, definition in defs.items() if name in reached}
return {**body, "$defs": kept} if kept else body
def _mergeable_branch(
root: Mapping[str, object],
branch: object,

View file

@ -2838,7 +2838,98 @@ def test_bedrock_tools_pt_follows_a_chain_of_root_refs_for_claude() -> None:
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
assert sent == {**_ADDRESS_SCHEMA, "$defs": defs}
assert sent == _ADDRESS_SCHEMA
_ORDER_SCHEMA: Final = {
"type": "object",
"properties": {"street": {"type": "string"}, "city": {"type": "string"}},
"required": ["street", "city"],
}
@pytest.mark.parametrize(
("siblings", "expected"),
[
(
{"type": "object", "properties": {}, "additionalProperties": False},
{**_ORDER_SCHEMA, "additionalProperties": False},
),
(
{
"properties": {"city": {"type": "string", "enum": ["Paris"]}, "note": {"type": "string"}},
"required": ["note"],
},
{
"type": "object",
"properties": {
"street": {"type": "string"},
"city": {"type": "string", "enum": ["Paris"]},
"note": {"type": "string"},
},
"required": ["street", "city", "note"],
},
),
],
ids=["mcp_normalized_empty_properties", "root_properties_win"],
)
def test_bedrock_tools_pt_merges_root_ref_siblings_into_its_target_for_claude(
siblings: dict[str, object], expected: dict[str, object]
) -> None:
parameters: Final = {"$ref": "#/$defs/Order", "$defs": {"Order": _ORDER_SCHEMA}, **siblings}
tools: Final = [{"type": "function", "function": {"name": "ship_order", "strict": True, "parameters": parameters}}]
tool_spec: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-haiku-4-5-20251001-v1:0")[0]["toolSpec"]
assert tool_spec["inputSchema"]["json"] == expected
def test_bedrock_tools_pt_sends_claude_only_the_defs_its_refs_reach() -> None:
defs: Final = {
"Home": {"type": "object", "properties": {"address": {"$ref": "#/$defs/Address"}}},
"Address": _ADDRESS_SCHEMA,
"Place": _ORDER_SCHEMA,
"Unused": {"type": "string"},
}
properties: Final = {
"home": {"$ref": "#/$defs/Home"},
"city": {"$ref": "#/$defs/Place/properties/city"},
"same_home": {"$ref": "#/properties/home"},
"parent": {"$ref": "#"},
}
parameters: Final = {"type": "object", "properties": properties, "$defs": defs}
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
assert sent["$defs"] == {"Home": defs["Home"], "Address": _ADDRESS_SCHEMA, "Place": _ORDER_SCHEMA}
def test_bedrock_tools_pt_keeps_a_claude_def_whose_name_its_ref_escapes() -> None:
defs: Final = {"a/b c~1": _ADDRESS_SCHEMA, "Unused": {"type": "string"}}
parameters: Final = {"type": "object", "properties": {"home": {"$ref": "#/$defs/a~1b%20c~01"}}, "$defs": defs}
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
assert sent["$defs"] == {"a/b c~1": _ADDRESS_SCHEMA}
@pytest.mark.parametrize(
"home",
[{"$ref": "#home"}, {"$dynamicRef": "#home"}, {"$ref": "#/$defs"}],
ids=["anchor_ref", "dynamic_ref", "whole_defs_ref"],
)
def test_bedrock_tools_pt_keeps_every_claude_def_when_a_ref_may_reach_one_by_other_means(
home: dict[str, object],
) -> None:
defs: Final = {"Home": {"$anchor": "home", **_ADDRESS_SCHEMA}, "Spare": {"type": "string"}}
parameters: Final = {"type": "object", "properties": {"home": home}, "$defs": defs}
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
assert sent["$defs"] == defs
def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() -> None:
@ -2850,8 +2941,7 @@ def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude()
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
home: Final = {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]}
assert sent == {**home, "$defs": {"Home": home}}
assert sent == {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]}
def test_bedrock_tools_pt_inlines_only_legacy_refs_for_claude() -> None:

View file

@ -27,7 +27,7 @@ from litellm.llms.anthropic.chat.transformation import (
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS,
NATIVE_OUTPUT_MAX_UNION_PARAMETERS,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import BEDROCK_TOOLS_MAX_INLINED_BYTES
from litellm.litellm_core_utils.prompt_templates.common_utils import DEFS_MAX_INLINED_BYTES
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
@ -5787,8 +5787,21 @@ def test_tool_defs_that_double_at_every_level_fail_fast_with_a_400_naming_the_to
mock_post.assert_not_called()
def test_tool_defs_that_inline_to_megabytes_still_reach_models_that_need_inlining():
description: Final = "k" * 3_000_000
parameters: Final = {
"type": "object",
"properties": {"key": {"$ref": "#/$defs/Key"}},
"$defs": {"Key": {"type": "string", "description": description}},
}
sent: Final = _bedrock_tools_pt([_tool("lookup", parameters)], model="us.amazon.nova-2-lite-v1:0")
assert sent[0]["toolSpec"]["inputSchema"]["json"]["properties"]["key"]["description"] == description
def test_tool_defs_inlining_budget_covers_every_tool_in_the_request():
description_bytes: Final = BEDROCK_TOOLS_MAX_INLINED_BYTES * 2 // 5
description_bytes: Final = DEFS_MAX_INLINED_BYTES * 2 // 5
tools: Final = [
_tool(
f"lookup_{index}",