From 60729aa3bea7fbcf68c4b52ea77dc67bf1c0916a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:48:36 -0800 Subject: [PATCH] fix(bedrock): keep shadowed and unused Claude defs from changing meaning or spending budget, and inline boolean definitions --- .../prompt_templates/common_utils.py | 35 +++++++--- .../prompt_templates/factory.py | 2 +- .../prompt_templates/json_schema_utils.py | 9 ++- litellm/llms/bedrock/common_utils.py | 2 +- ...ore_utils_prompt_templates_common_utils.py | 35 ++++++++++ ...llm_core_utils_prompt_templates_factory.py | 66 +++++++++++++++++++ 6 files changed, 137 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index e45ca4ace4a..85b071f8938 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -950,7 +950,13 @@ def _resolve_in_scope(scope: _DefsScope, name: str) -> object: return next((defs[name] for defs in scope if name in defs), None) -def _keeps_no_ref(_ref: str) -> bool: +def _definition_schema(definition: object) -> Mapping[str, object] | None: + if isinstance(definition, bool): + return {} if definition else {"not": {}} + return definition if isinstance(definition, dict) else None + + +def _keeps_no_ref(_ref: str, _scope: _DefsScope) -> bool: return False @@ -959,7 +965,7 @@ def unpack_defs( defs: dict, max_inlined_bytes: int | None = None, *, - keeps_ref: Callable[[str], bool] = _keeps_no_ref, + keeps_ref: Callable[[str, _DefsScope], bool] = _keeps_no_ref, ) -> int: """Expand *all* ``$ref`` entries pointing into ``$defs`` / ``definitions``. @@ -981,8 +987,8 @@ def unpack_defs( Defaults to ``None`` (unbounded) so existing callers are unaffected; raises ``ValueError`` on overflow. - ``keeps_ref`` names the refs to leave as they are, for a provider that - resolves those itself. + ``keeps_ref`` names the refs to leave as they are, given the definition + layers in scope, nearest first, for a provider that resolves those itself. """ import copy @@ -1007,7 +1013,7 @@ def unpack_defs( if isinstance(node, dict): # --- Case 1: this node *is* a reference --- ref = node.get("$ref") - if isinstance(ref, str) and not keeps_ref(ref): + if isinstance(ref, str) and not keeps_ref(ref, active_defs): ref_name = ref.split("/")[-1] # Check for circular reference in the resolution chain @@ -1015,9 +1021,9 @@ def unpack_defs( # Circular reference detected - leave as-is to prevent infinite recursion continue - target_schema = _resolve_in_scope(active_defs, ref_name) + target_schema = _definition_schema(_resolve_in_scope(active_defs, ref_name)) # Unknown reference – leave untouched - if not isinstance(target_schema, dict): + if target_schema is None: continue if max_inlined_bytes is not None: @@ -1086,6 +1092,15 @@ def _legacy_defs(schema: object) -> Mapping[str, object] | None: return {**_mapping_or_empty(components.get("schemas")), **_mapping_or_empty(node.get("definitions"))} +def _legacy_blocks_are_well_formed(schema: Mapping[str, object]) -> bool: + components: Final = schema.get("components", {}) + return ( + isinstance(schema.get("definitions") or {}, dict) + and isinstance(components, dict) + and isinstance(_mapping_or_empty(components).get("schemas") or {}, dict) + ) + + # Schema-bomb budget for ``$ref`` inlining: cap the cumulative JSON-byte # size of every inlined target. A byte cap is the universal measure of # expansion -- it simultaneously bounds ref-count fan-out, node-count @@ -1116,7 +1131,7 @@ def unpack_legacy_defs( schemas cannot expand into a schema-bomb before reaching the upstream provider -- raises ``ValueError`` on overflow. """ - if _legacy_defs(schema) is None: + if _legacy_defs(schema) is None or not _legacy_blocks_are_well_formed(schema): return schema if copy: import copy as _copy @@ -1139,7 +1154,9 @@ def inline_refs_outside_root_defs( document, {**root_defs, **(_legacy_defs(schema) or {})}, max_inlined_bytes=max_inlined_bytes, - keeps_ref=lambda ref: ref_points_into_defs(ref, root_defs), + keeps_ref=lambda ref, scope: ( + ref_points_into_defs(ref, root_defs) and not any(ref_points_into_defs(ref, layer) for layer in scope[:-1]) + ), ) inlined: Final = document["body"] return ({**inlined, "$defs": root_defs} if root_defs else inlined), inlined_bytes diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5d411461084..fe5ff921e95 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5953,7 +5953,7 @@ def _bedrock_tools_pt( try: if resolves_schema_defs: inlined, tool_inlined_bytes = inline_refs_outside_root_defs( - schema_with_root_ref_resolved(parameters), remaining_inline_bytes + schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters)), remaining_inline_bytes ) parameters = dict(schema_without_unreferenced_defs(inlined)) else: diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py index a67e39ead00..561cedf7e94 100644 --- a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -106,10 +106,17 @@ def ref_points_into_defs(ref: str, defs: Mapping[str, object]) -> bool: return names is not None and any(name in defs for name in names) +def _def_names_a_ref_may_reach(ref: object) -> frozenset[str] | None: + names: Final = _referenced_def_names(ref) + if names is None or not isinstance(ref, str): + return names + return names | {ref.split("/")[-1]} + + def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]: if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS): return (None,) - return (_referenced_def_names(node["$ref"]),) if "$ref" in node else () + return (_def_names_a_ref_may_reach(node["$ref"]),) if "$ref" in node else () def _json_object(value: object) -> Mapping[str, object] | None: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f25cab5d782..e6fd92ece63 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1609,7 +1609,7 @@ def _is_claude(model: str) -> bool: def bedrock_converse_resolves_tool_schema_defs(model: str | None) -> bool: - return model is not None and _is_claude(model) + return model is not None and _is_claude_behind_any_region_prefix(model) def _is_claude_behind_any_region_prefix(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 297e8db5e92..bd454a9d53c 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -342,6 +342,41 @@ class TestUnpackLegacyDefs: assert out["properties"]["home"] == address + def test_a_boolean_definition_is_inlined_as_the_schema_it_means(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + unpack_legacy_defs, + ) + + schema: Final = { + "type": "object", + "properties": {"anything": {"$ref": "#/definitions/Anything"}, "nothing": {"$ref": "#/definitions/Nothing"}}, + "definitions": {"Anything": True, "Nothing": False}, + } + + out: Final = unpack_legacy_defs(schema) + + assert out == {"type": "object", "properties": {"anything": {}, "nothing": {"not": {}}}} + + @pytest.mark.parametrize( + "legacy_blocks", + [ + {"definitions": 5}, + {"definitions": {"A": {"type": "string"}}, "components": {"schemas": ["not", "a", "map"]}}, + {"definitions": {"A": {"type": "string"}}, "components": 5}, + ], + ids=["definitions_is_a_number", "components_schemas_is_a_list", "components_is_a_number"], + ) + def test_a_malformed_legacy_block_is_left_for_the_provider_to_judge(self, legacy_blocks): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + unpack_legacy_defs, + ) + + schema: Final = {"type": "object", "properties": {"a": {"$ref": "#/definitions/A"}}, **legacy_blocks} + + out: Final = unpack_legacy_defs(copy.deepcopy(schema)) + + assert out == schema + def test_memory_stays_linear_in_schema_size_with_many_definitions(self): from litellm.litellm_core_utils.prompt_templates.common_utils import ( unpack_legacy_defs, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 2abd55d4b94..45174e108d5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2944,6 +2944,7 @@ def _with_root_defs_inlined(schema: object, defs: Mapping[str, object]) -> objec _SHIP_TO: Final = {"type": "object", "properties": {"to": {"$ref": "#/$defs/Address"}}, "required": ["to"]} +_ZIP_ONLY: Final = {"type": "object", "properties": {"zip": {"type": "string"}}, "required": ["zip"]} @pytest.mark.parametrize( @@ -2957,6 +2958,12 @@ _SHIP_TO: Final = {"type": "object", "properties": {"to": {"$ref": "#/$defs/Addr "required": ["to"], "$defs": {"Address": _ADDRESS_SCHEMA}, }, + { + "type": "object", + "properties": {"to": {"$ref": "#/definitions/Address"}}, + "required": ["to"], + "$defs": {"Address": _ADDRESS_SCHEMA}, + }, { "type": "object", "properties": {"ship": {**_SHIP_TO, "$defs": {"Address": _ADDRESS_SCHEMA}}}, @@ -2974,13 +2981,28 @@ _SHIP_TO: Final = {"type": "object", "properties": {"to": {"$ref": "#/$defs/Addr }, "$defs": {"Address": _ADDRESS_SCHEMA}, }, + { + "type": "object", + "properties": {"ship": {**_SHIP_TO, "$defs": {"Address": _ZIP_ONLY}}}, + "required": ["ship"], + "$defs": {"Address": _ADDRESS_SCHEMA}, + }, + { + "type": "object", + "properties": {"ship": {"$ref": "#/$defs/Ship"}}, + "required": ["ship"], + "$defs": {"Ship": {**_SHIP_TO, "$defs": {"Address": _ZIP_ONLY}}, "Address": _ADDRESS_SCHEMA}, + }, ], ids=[ "root_ref_into_definitions_beside_defs", "root_ref_target_with_its_own_defs", "percent_encoded_defs_pointer", + "definitions_pointer_into_root_defs", "property_with_its_own_defs", "ref_under_dependencies", + "property_defs_shadowing_a_root_name", + "defs_entry_shadowing_a_root_name", ], ) def test_bedrock_tools_pt_sends_claude_a_schema_that_means_what_full_inlining_does( @@ -2996,6 +3018,50 @@ def test_bedrock_tools_pt_sends_claude_a_schema_that_means_what_full_inlining_do assert _with_root_defs_inlined(claude_body, claude_schema.get("$defs", {})) == inlined["inputSchema"]["json"] +def test_bedrock_tools_pt_spends_no_inlining_budget_on_unused_claude_defs() -> None: + big: Final = {"type": "string", "description": "x" * 1_000_000} + unused: Final = {f"Unused{i}": {"$ref": "#/definitions/Big"} for i in range(11)} + parameters: Final = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "$defs": unused, + "definitions": {"Big": big}, + } + tools: Final = [{"type": "function", "function": {"name": "t", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent == {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]} + + +def test_bedrock_tools_pt_inlines_boolean_definitions_as_the_schemas_they_mean() -> None: + parameters: Final = { + "type": "object", + "properties": {"anything": {"$ref": "#/$defs/Anything"}, "nothing": {"$ref": "#/$defs/Nothing"}}, + "$defs": {"Anything": True, "Nothing": False}, + } + tools: Final = [{"type": "function", "function": {"name": "t", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.amazon.nova-2-lite-v1:0")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["properties"] == {"anything": {}, "nothing": {"not": {}}} + + +def test_bedrock_tools_pt_keeps_defs_for_claude_behind_any_region_prefix() -> None: + parameters: Final = { + "type": "object", + "properties": {"to": {"$ref": "#/$defs/Address"}}, + "required": ["to"], + "$defs": {"Address": _ADDRESS_SCHEMA}, + } + tools: Final = [{"type": "function", "function": {"name": "ship", "parameters": deepcopy(parameters)}}] + + sent: Final = _bedrock_tools_pt(tools, model="ca.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent == parameters + + def test_bedrock_tools_pt_sends_no_defs_to_models_that_need_inlining() -> None: parameters: Final = {"$ref": "#/$defs/Ship", "$defs": {"Ship": {**_SHIP_TO, "$defs": {"Address": _ADDRESS_SCHEMA}}}} tools: Final = [{"type": "function", "function": {"name": "ship", "parameters": parameters}}]