diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 4e93a6729ab..e45ca4ace4a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import importlib import json import sys import warnings -from collections.abc import Hashable, Iterable, Mapping, Sequence +from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence from dataclasses import dataclass from itertools import chain, groupby from types import MappingProxyType, ModuleType @@ -57,6 +57,7 @@ from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( is_object_schema as is_object_schema, # noqa: PLC0414 # public compatibility re-export ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ref_points_into_defs from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( schema_fragment_ref as schema_fragment_ref, # noqa: PLC0414 # public compatibility re-export ) @@ -930,16 +931,15 @@ def _estimate_json_bytes(obj: object) -> int: _DefsScope: TypeAlias = tuple[Mapping[str, object], ...] _DEFS_KEYS: Final = ("$defs", "definitions") -_LEGACY_DEFS_KEYS: Final = ("definitions",) -_LEGACY_REF_PREFIXES: Final = ("#/definitions/", "#/components/schemas/") +_NON_BODY_KEYS: Final = frozenset({"$defs", "definitions", "components"}) def _mapping_or_empty(value: object) -> Mapping[str, object]: return value if isinstance(value, dict) else {} -def _local_defs(node: Mapping[str, object], defs_keys: tuple[str, ...]) -> Mapping[str, object]: - return dict(chain.from_iterable(_mapping_or_empty(node.get(key)).items() for key in defs_keys)) +def _local_defs(node: Mapping[str, object]) -> Mapping[str, object]: + return dict(chain.from_iterable(_mapping_or_empty(node.get(key)).items() for key in _DEFS_KEYS)) def _scope_with_local_defs(scope: _DefsScope, local_defs: Mapping[str, object]) -> _DefsScope: @@ -950,12 +950,16 @@ def _resolve_in_scope(scope: _DefsScope, name: str) -> object: return next((defs[name] for defs in scope if name in defs), None) +def _keeps_no_ref(_ref: str) -> bool: + return False + + def unpack_defs( schema: dict, defs: dict, max_inlined_bytes: int | None = None, *, - legacy_only: bool = False, + keeps_ref: Callable[[str], bool] = _keeps_no_ref, ) -> int: """Expand *all* ``$ref`` entries pointing into ``$defs`` / ``definitions``. @@ -977,8 +981,8 @@ def unpack_defs( Defaults to ``None`` (unbounded) so existing callers are unaffected; raises ``ValueError`` on overflow. - ``legacy_only`` resolves only ``definitions`` / ``components.schemas`` - refs and leaves ``$defs`` refs for a provider that resolves them itself. + ``keeps_ref`` names the refs to leave as they are, for a provider that + resolves those itself. """ import copy @@ -987,8 +991,7 @@ def unpack_defs( # Combine the defs handed down by the caller with defs/definitions found on # the current node. Local keys shadow parent keys to match JSON-schema # scoping rules. - defs_keys: Final = _LEGACY_DEFS_KEYS if legacy_only else _DEFS_KEYS - root_defs: Final[_DefsScope] = ({**defs, **_local_defs(schema, defs_keys)},) + root_defs: Final[_DefsScope] = ({**defs, **_local_defs(schema)},) # Use iterative approach with queue to avoid recursion # Each item in queue is (node, parent_container, key/index, active_defs, ref_chain) @@ -1004,7 +1007,7 @@ def unpack_defs( if isinstance(node, dict): # --- Case 1: this node *is* a reference --- ref = node.get("$ref") - if isinstance(ref, str) and (not legacy_only or ref.startswith(_LEGACY_REF_PREFIXES)): + if isinstance(ref, str) and not keeps_ref(ref): ref_name = ref.split("/")[-1] # Check for circular reference in the resolution chain @@ -1028,7 +1031,7 @@ def unpack_defs( ) # Merge defs from the target to capture nested definitions - child_defs = _scope_with_local_defs(active_defs, _local_defs(target_schema, defs_keys)) + child_defs = _scope_with_local_defs(active_defs, _local_defs(target_schema)) # Replace the reference with resolved copy resolved = copy.deepcopy(target_schema) @@ -1056,7 +1059,7 @@ def unpack_defs( # --- Case 2: regular dict – process its values --- # Update defs with any nested $defs/definitions present *here*. - current_defs = _scope_with_local_defs(active_defs, _local_defs(node, defs_keys)) + current_defs = _scope_with_local_defs(active_defs, _local_defs(node)) # Add all dict values to queue for k, v in node.items(): @@ -1119,17 +1122,27 @@ def unpack_legacy_defs( import copy as _copy schema = _copy.deepcopy(schema) - inline_legacy_defs(schema, max_inlined_bytes) + defs: Final = _legacy_defs(schema) or {} + schema.pop("components", None) + schema.pop("definitions", None) + unpack_defs(schema, dict(defs), max_inlined_bytes=max_inlined_bytes) return schema -def inline_legacy_defs(schema: dict, max_inlined_bytes: int) -> int: - defs: Final = _legacy_defs(schema) - if defs is None: - return 0 - schema.pop("components", None) - schema.pop("definitions", None) - return unpack_defs(schema, dict(defs), max_inlined_bytes=max_inlined_bytes, legacy_only=True) +def inline_refs_outside_root_defs( + schema: Mapping[str, object], max_inlined_bytes: int +) -> tuple[Mapping[str, object], int]: + root_defs: Final = dict(_mapping_or_empty(schema.get("$defs"))) + body: Final = {key: value for key, value in schema.items() if key not in _NON_BODY_KEYS} + document: Final = {"body": body, "defs": root_defs} + inlined_bytes: Final = unpack_defs( + document, + {**root_defs, **(_legacy_defs(schema) or {})}, + max_inlined_bytes=max_inlined_bytes, + keeps_ref=lambda ref: ref_points_into_defs(ref, root_defs), + ) + inlined: Final = document["body"] + return ({**inlined, "$defs": root_defs} if root_defs else inlined), inlined_bytes def sanitize_input_schema_for_anthropic(input_schema: dict) -> "AnthropicInputSchema": diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 7193659cd53..5d411461084 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -71,7 +71,7 @@ from .common_utils import ( convert_content_list_to_str, has_blank_redacted_data, infer_content_type_from_url_and_content, - inline_legacy_defs, + inline_refs_outside_root_defs, is_encrypted_reasoning_block, is_non_content_values_set, is_unsignable_thinking_block, @@ -5952,8 +5952,10 @@ def _bedrock_tools_pt( remaining_inline_bytes = DEFS_MAX_INLINED_BYTES - inlined_bytes try: if resolves_schema_defs: - tool_inlined_bytes = inline_legacy_defs(parameters, remaining_inline_bytes) - parameters = dict(schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters))) + inlined, tool_inlined_bytes = inline_refs_outside_root_defs( + schema_with_root_ref_resolved(parameters), remaining_inline_bytes + ) + parameters = dict(schema_without_unreferenced_defs(inlined)) else: defs = parameters.pop("$defs", {}) defs_copy = defs if isinstance(defs, dict) else {} @@ -5962,6 +5964,7 @@ def _bedrock_tools_pt( # with circular references (see issue #19098). unpack_defs handles nested # refs recursively and correctly detects/skips circular references. tool_inlined_bytes = unpack_defs(parameters, defs_copy, max_inlined_bytes=remaining_inline_bytes) + parameters.pop("$defs", None) except ValueError as e: raise BedrockError( status_code=400, diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py index de00f02ea6c..a67e39ead00 100644 --- a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -16,6 +16,7 @@ _MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 _EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) _ROOT_DEFS_POINTER: Final = "#/$defs/" _DYNAMIC_REF_KEYWORDS: Final = ("$dynamicRef", "$recursiveRef") +_DEFS_CONTAINERS: Final = ("$defs", "definitions") def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: @@ -70,9 +71,15 @@ def _with_root_ref_inlined(schema: Mapping[str, object]) -> Mapping[str, object] return schema siblings: Final = {key: value for key, value in schema.items() if key != "$ref"} required: Final = dict.fromkeys((*_required_names_in_order(target), *_required_names_in_order(schema))) + defs: Final = { + container: {**(_schema_at(target, container) or {}), **(_schema_at(schema, container) or {})} + for container in _DEFS_CONTAINERS + if container in target or container in schema + } return { **target, **siblings, + **defs, "properties": {**_schema_properties(target), **_schema_properties(schema)}, "required": list(required), } @@ -94,14 +101,42 @@ def _referenced_def_names(ref: object) -> frozenset[str] | None: return frozenset((name, unquote(name).replace("~1", "/").replace("~0", "~"))) +def ref_points_into_defs(ref: str, defs: Mapping[str, object]) -> bool: + names: Final = _referenced_def_names(ref) + return names is not None and any(name in defs for name in names) + + def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]: if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS): return (None,) return (_referenced_def_names(node["$ref"]),) if "$ref" in node else () +def _json_object(value: object) -> Mapping[str, object] | None: + return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None + + +def _json_children(value: object) -> tuple[object, ...]: + mapping: Final = _json_object(value) + if mapping is not None: + return tuple(mapping.values()) + return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () + + +def _json_levels(value: object) -> Iterator[tuple[object, ...]]: + frontier: tuple[object, ...] = (value,) # rebind-ok: level-order cursor, one level a round + while frontier: + yield frontier + frontier = tuple(chain.from_iterable(_json_children(node) for node in frontier)) + + +def _json_objects(value: object) -> Iterator[Mapping[str, object]]: + values: Final = chain.from_iterable(islice(_json_levels(value), _MAX_SCHEMA_NESTING)) + return (node for item in values if (node := _json_object(item)) is not None) + + def _schema_ref_def_names(schema: Mapping[str, object]) -> Iterator[frozenset[str] | None]: - return chain.from_iterable(_node_ref_def_names(node) for node in _schema_nodes(schema)) + return chain.from_iterable(_node_ref_def_names(node) for node in _json_objects(schema)) def _reached_def_names(body: Mapping[str, object], defs: Mapping[str, object]) -> frozenset[str] | None: @@ -379,6 +414,7 @@ def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object] _SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 4ba026ed366..297e8db5e92 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -326,6 +326,22 @@ class TestUnpackLegacyDefs: assert "components" not in out assert out["properties"]["r0"]["properties"]["p0"] == {"type": "string"} + def test_a_defs_ref_is_inlined_from_the_legacy_block_beside_it(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + unpack_legacy_defs, + ) + + address: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + schema: Final = { + "type": "object", + "properties": {"home": {"$ref": "#/$defs/Address"}}, + "definitions": {"Address": address}, + } + + out: Final = unpack_legacy_defs(schema) + + assert out["properties"]["home"] == address + def test_memory_stays_linear_in_schema_size_with_many_definitions(self): from litellm.litellm_core_utils.prompt_templates.common_utils import ( unpack_legacy_defs, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 21d67cedf30..2abd55d4b94 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2932,6 +2932,79 @@ def test_bedrock_tools_pt_keeps_every_claude_def_when_a_ref_may_reach_one_by_oth assert sent["$defs"] == defs +def _with_root_defs_inlined(schema: object, defs: Mapping[str, object]) -> object: + if isinstance(schema, list): + return [_with_root_defs_inlined(item, defs) for item in schema] + if not isinstance(schema, dict): + return schema + ref: Final = schema.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + return _with_root_defs_inlined(defs[ref.removeprefix("#/$defs/")], defs) + return {key: _with_root_defs_inlined(value, defs) for key, value in schema.items()} + + +_SHIP_TO: Final = {"type": "object", "properties": {"to": {"$ref": "#/$defs/Address"}}, "required": ["to"]} + + +@pytest.mark.parametrize( + "parameters", + [ + {"$ref": "#/definitions/Ship", "definitions": {"Ship": _SHIP_TO}, "$defs": {"Address": _ADDRESS_SCHEMA}}, + {"$ref": "#/$defs/Ship", "$defs": {"Ship": {**_SHIP_TO, "$defs": {"Address": _ADDRESS_SCHEMA}}}}, + { + "type": "object", + "properties": {"to": {"$ref": "#/%24defs/Address"}}, + "required": ["to"], + "$defs": {"Address": _ADDRESS_SCHEMA}, + }, + { + "type": "object", + "properties": {"ship": {**_SHIP_TO, "$defs": {"Address": _ADDRESS_SCHEMA}}}, + "required": ["ship"], + "$defs": {"Unused": {"type": "string"}}, + }, + { + "type": "object", + "properties": { + "ship": { + "type": "object", + "properties": {"gift": {"type": "boolean"}}, + "dependencies": {"gift": {"properties": {"to": {"$ref": "#/$defs/Address"}}}}, + } + }, + "$defs": {"Address": _ADDRESS_SCHEMA}, + }, + ], + ids=[ + "root_ref_into_definitions_beside_defs", + "root_ref_target_with_its_own_defs", + "percent_encoded_defs_pointer", + "property_with_its_own_defs", + "ref_under_dependencies", + ], +) +def test_bedrock_tools_pt_sends_claude_a_schema_that_means_what_full_inlining_does( + parameters: dict[str, object], +) -> None: + tools: Final = [{"type": "function", "function": {"name": "ship", "parameters": parameters}}] + + claude: Final = _bedrock_tools_pt(deepcopy(tools), model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"] + inlined: Final = _bedrock_tools_pt(deepcopy(tools), model="us.amazon.nova-2-lite-v1:0")[0]["toolSpec"] + + claude_schema: Final = claude["inputSchema"]["json"] + claude_body: Final = {key: value for key, value in claude_schema.items() if key != "$defs"} + assert _with_root_defs_inlined(claude_body, claude_schema.get("$defs", {})) == inlined["inputSchema"]["json"] + + +def test_bedrock_tools_pt_sends_no_defs_to_models_that_need_inlining() -> None: + parameters: Final = {"$ref": "#/$defs/Ship", "$defs": {"Ship": {**_SHIP_TO, "$defs": {"Address": _ADDRESS_SCHEMA}}}} + tools: Final = [{"type": "function", "function": {"name": "ship", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.amazon.nova-2-lite-v1:0")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent == {"type": "object", "properties": {"to": _ADDRESS_SCHEMA}, "required": ["to"]} + + def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() -> None: defs: Final = { "Home": {"type": "object", "properties": {"address": {"$ref": "#/definitions/Address"}}, "required": ["address"]}