From 6fc409da1aa40da42870291fd2a48b031dcc6997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:41:58 -0800 Subject: [PATCH] fix(bedrock): merge root ref siblings, drop unused $defs, and use the shared inlining budget A root $ref beside an empty properties block, the shape MCP tools get, now merges into its target instead of replacing the target's properties, so a strict Claude tool sends its real arguments again. Claude also stops getting $defs entries no $ref reaches, which kept unused definitions out of its input tokens before this branch. Inlining for other models and for legacy definitions now uses the same 10 MB budget the native Anthropic route uses, since 1 MB refused an 852 KB expansion that worked at the merge base, and the 400 now names the bytes of definitions copied --- .../prompt_templates/common_utils.py | 1 - .../prompt_templates/factory.py | 13 +-- .../prompt_templates/json_schema_utils.py | 63 +++++++++++- ...llm_core_utils_prompt_templates_factory.py | 96 ++++++++++++++++++- .../chat/test_converse_transformation.py | 17 +++- 5 files changed, 177 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 40f86905ed0..4e93a6729ab 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1092,7 +1092,6 @@ def _legacy_defs(schema: object) -> Mapping[str, object] | None: # below memory-pressure territory, and rejects request-supplied bombs before # the proxy materialises them. DEFS_MAX_INLINED_BYTES: Final = 10_000_000 -BEDROCK_TOOLS_MAX_INLINED_BYTES: Final = 1_000_000 def unpack_legacy_defs( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index a7d7e97250f..7193659cd53 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -65,7 +65,7 @@ from litellm.types.llms.vertex_ai import PartType as VertexPartType from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( - BEDROCK_TOOLS_MAX_INLINED_BYTES, + DEFS_MAX_INLINED_BYTES, as_openai_image_part, converse_redacted_content, convert_content_list_to_str, @@ -81,7 +81,7 @@ from .common_utils import ( unpack_defs, ) from .image_handling import convert_url_to_base64, is_pdf_media_type -from .json_schema_utils import schema_with_root_ref_resolved +from .json_schema_utils import schema_with_root_ref_resolved, schema_without_unreferenced_defs def default_pt(messages): @@ -5949,11 +5949,11 @@ def _bedrock_tools_pt( else: description = name - remaining_inline_bytes = BEDROCK_TOOLS_MAX_INLINED_BYTES - inlined_bytes + remaining_inline_bytes = DEFS_MAX_INLINED_BYTES - inlined_bytes try: if resolves_schema_defs: tool_inlined_bytes = inline_legacy_defs(parameters, remaining_inline_bytes) - parameters = dict(schema_with_root_ref_resolved(parameters)) + parameters = dict(schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters))) else: defs = parameters.pop("$defs", {}) defs_copy = defs if isinstance(defs, dict) else {} @@ -5965,8 +5965,9 @@ def _bedrock_tools_pt( except ValueError as e: raise BedrockError( status_code=400, - message=f"Inlining the $ref definitions of tool '{raw_name}' pushes the request's tool schemas past " - f"{BEDROCK_TOOLS_MAX_INLINED_BYTES:,} bytes. Reference shared $defs less often or flatten the schema.", + message=f"Inlining the $ref definitions of tool '{raw_name}' copies more than " + f"{DEFS_MAX_INLINED_BYTES:,} bytes of definitions across the request's tools. " + "Reference shared $defs less often or flatten the schema.", ) from e inlined_bytes += tool_inlined_bytes normalize_json_schema_custom_types_to_object(parameters) diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py index 385684fe586..de00f02ea6c 100644 --- a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -5,6 +5,7 @@ from graphlib import CycleError, TopologicalSorter from itertools import chain, islice from types import MappingProxyType from typing import Final +from urllib.parse import unquote from pydantic import TypeAdapter @@ -13,6 +14,8 @@ _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") _LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) _MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 _EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) +_ROOT_DEFS_POINTER: Final = "#/$defs/" +_DYNAMIC_REF_KEYWORDS: Final = ("$dynamicRef", "$recursiveRef") def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: @@ -56,18 +59,76 @@ def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[s return target if isinstance(target, dict) else None +def _required_names_in_order(schema: Mapping[str, object]) -> tuple[str, ...]: + return tuple(name for name in _list_at(schema, "required") if isinstance(name, str)) + + def _with_root_ref_inlined(schema: Mapping[str, object]) -> Mapping[str, object]: ref: Final = schema.get("$ref") target: Final = _resolve_local_schema_ref(schema, ref) if isinstance(ref, str) else None if target is None: return schema - return {**target, **{key: value for key, value in schema.items() if key != "$ref"}} + siblings: Final = {key: value for key, value in schema.items() if key != "$ref"} + required: Final = dict.fromkeys((*_required_names_in_order(target), *_required_names_in_order(schema))) + return { + **target, + **siblings, + "properties": {**_schema_properties(target), **_schema_properties(schema)}, + "required": list(required), + } def schema_with_root_ref_resolved(schema: Mapping[str, object]) -> Mapping[str, object]: return reduce(lambda resolved, _hop: _with_root_ref_inlined(resolved), range(_MAX_SCHEMA_FLATTEN_DEPTH), schema) +def _referenced_def_names(ref: object) -> frozenset[str] | None: + if not isinstance(ref, str): + return frozenset() + pointer: Final = schema_fragment_ref(ref) + if pointer == "#" or (pointer.startswith("#/") and not pointer.startswith("#/$defs")): + return frozenset() + if not pointer.startswith(_ROOT_DEFS_POINTER): + return None + name: Final = pointer[len(_ROOT_DEFS_POINTER) :].split("/")[0] + return frozenset((name, unquote(name).replace("~1", "/").replace("~0", "~"))) + + +def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]: + if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS): + return (None,) + return (_referenced_def_names(node["$ref"]),) if "$ref" in node else () + + +def _schema_ref_def_names(schema: Mapping[str, object]) -> Iterator[frozenset[str] | None]: + return chain.from_iterable(_node_ref_def_names(node) for node in _schema_nodes(schema)) + + +def _reached_def_names(body: Mapping[str, object], defs: Mapping[str, object]) -> frozenset[str] | None: + reached: frozenset[str] = frozenset() # rebind-ok: grows by the definitions the last round's refs name + frontier: tuple[Mapping[str, object], ...] = (body,) # rebind-ok: the schemas the last round reached + while frontier: + found = tuple(chain.from_iterable(_schema_ref_def_names(node) for node in frontier)) + if None in found: + return None + new_names = frozenset(chain.from_iterable(names for names in found if names is not None)) - reached + reached = reached | new_names + frontier = tuple(definition for name in new_names if (definition := _schema_at(defs, name)) is not None) + return reached + + +def schema_without_unreferenced_defs(schema: Mapping[str, object]) -> Mapping[str, object]: + defs: Final = _schema_at(schema, "$defs") + if defs is None: + return schema + body: Final = {key: value for key, value in schema.items() if key != "$defs"} + reached: Final = _reached_def_names(body, defs) + if reached is None: + return schema + kept: Final = {name: definition for name, definition in defs.items() if name in reached} + return {**body, "$defs": kept} if kept else body + + def _mergeable_branch( root: Mapping[str, object], branch: object, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 27d967270f1..21d67cedf30 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2838,7 +2838,98 @@ def test_bedrock_tools_pt_follows_a_chain_of_root_refs_for_claude() -> None: sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] - assert sent == {**_ADDRESS_SCHEMA, "$defs": defs} + assert sent == _ADDRESS_SCHEMA + + +_ORDER_SCHEMA: Final = { + "type": "object", + "properties": {"street": {"type": "string"}, "city": {"type": "string"}}, + "required": ["street", "city"], +} + + +@pytest.mark.parametrize( + ("siblings", "expected"), + [ + ( + {"type": "object", "properties": {}, "additionalProperties": False}, + {**_ORDER_SCHEMA, "additionalProperties": False}, + ), + ( + { + "properties": {"city": {"type": "string", "enum": ["Paris"]}, "note": {"type": "string"}}, + "required": ["note"], + }, + { + "type": "object", + "properties": { + "street": {"type": "string"}, + "city": {"type": "string", "enum": ["Paris"]}, + "note": {"type": "string"}, + }, + "required": ["street", "city", "note"], + }, + ), + ], + ids=["mcp_normalized_empty_properties", "root_properties_win"], +) +def test_bedrock_tools_pt_merges_root_ref_siblings_into_its_target_for_claude( + siblings: dict[str, object], expected: dict[str, object] +) -> None: + parameters: Final = {"$ref": "#/$defs/Order", "$defs": {"Order": _ORDER_SCHEMA}, **siblings} + tools: Final = [{"type": "function", "function": {"name": "ship_order", "strict": True, "parameters": parameters}}] + + tool_spec: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-haiku-4-5-20251001-v1:0")[0]["toolSpec"] + + assert tool_spec["inputSchema"]["json"] == expected + + +def test_bedrock_tools_pt_sends_claude_only_the_defs_its_refs_reach() -> None: + defs: Final = { + "Home": {"type": "object", "properties": {"address": {"$ref": "#/$defs/Address"}}}, + "Address": _ADDRESS_SCHEMA, + "Place": _ORDER_SCHEMA, + "Unused": {"type": "string"}, + } + properties: Final = { + "home": {"$ref": "#/$defs/Home"}, + "city": {"$ref": "#/$defs/Place/properties/city"}, + "same_home": {"$ref": "#/properties/home"}, + "parent": {"$ref": "#"}, + } + parameters: Final = {"type": "object", "properties": properties, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == {"Home": defs["Home"], "Address": _ADDRESS_SCHEMA, "Place": _ORDER_SCHEMA} + + +def test_bedrock_tools_pt_keeps_a_claude_def_whose_name_its_ref_escapes() -> None: + defs: Final = {"a/b c~1": _ADDRESS_SCHEMA, "Unused": {"type": "string"}} + parameters: Final = {"type": "object", "properties": {"home": {"$ref": "#/$defs/a~1b%20c~01"}}, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == {"a/b c~1": _ADDRESS_SCHEMA} + + +@pytest.mark.parametrize( + "home", + [{"$ref": "#home"}, {"$dynamicRef": "#home"}, {"$ref": "#/$defs"}], + ids=["anchor_ref", "dynamic_ref", "whole_defs_ref"], +) +def test_bedrock_tools_pt_keeps_every_claude_def_when_a_ref_may_reach_one_by_other_means( + home: dict[str, object], +) -> None: + defs: Final = {"Home": {"$anchor": "home", **_ADDRESS_SCHEMA}, "Spare": {"type": "string"}} + parameters: Final = {"type": "object", "properties": {"home": home}, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == defs def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() -> None: @@ -2850,8 +2941,7 @@ def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] - home: Final = {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]} - assert sent == {**home, "$defs": {"Home": home}} + assert sent == {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]} def test_bedrock_tools_pt_inlines_only_legacy_refs_for_claude() -> None: diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6de831dd02e..3be84e83bbf 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -27,7 +27,7 @@ from litellm.llms.anthropic.chat.transformation import ( NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, NATIVE_OUTPUT_MAX_UNION_PARAMETERS, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import BEDROCK_TOOLS_MAX_INLINED_BYTES +from litellm.litellm_core_utils.prompt_templates.common_utils import DEFS_MAX_INLINED_BYTES from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES @@ -5787,8 +5787,21 @@ def test_tool_defs_that_double_at_every_level_fail_fast_with_a_400_naming_the_to mock_post.assert_not_called() +def test_tool_defs_that_inline_to_megabytes_still_reach_models_that_need_inlining(): + description: Final = "k" * 3_000_000 + parameters: Final = { + "type": "object", + "properties": {"key": {"$ref": "#/$defs/Key"}}, + "$defs": {"Key": {"type": "string", "description": description}}, + } + + sent: Final = _bedrock_tools_pt([_tool("lookup", parameters)], model="us.amazon.nova-2-lite-v1:0") + + assert sent[0]["toolSpec"]["inputSchema"]["json"]["properties"]["key"]["description"] == description + + def test_tool_defs_inlining_budget_covers_every_tool_in_the_request(): - description_bytes: Final = BEDROCK_TOOLS_MAX_INLINED_BYTES * 2 // 5 + description_bytes: Final = DEFS_MAX_INLINED_BYTES * 2 // 5 tools: Final = [ _tool( f"lookup_{index}",