diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 40f86905ed0..4e93a6729ab 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1092,7 +1092,6 @@ def _legacy_defs(schema: object) -> Mapping[str, object] | None: # below memory-pressure territory, and rejects request-supplied bombs before # the proxy materialises them. DEFS_MAX_INLINED_BYTES: Final = 10_000_000 -BEDROCK_TOOLS_MAX_INLINED_BYTES: Final = 1_000_000 def unpack_legacy_defs( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index a7d7e97250f..7193659cd53 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -65,7 +65,7 @@ from litellm.types.llms.vertex_ai import PartType as VertexPartType from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( - BEDROCK_TOOLS_MAX_INLINED_BYTES, + DEFS_MAX_INLINED_BYTES, as_openai_image_part, converse_redacted_content, convert_content_list_to_str, @@ -81,7 +81,7 @@ from .common_utils import ( unpack_defs, ) from .image_handling import convert_url_to_base64, is_pdf_media_type -from .json_schema_utils import schema_with_root_ref_resolved +from .json_schema_utils import schema_with_root_ref_resolved, schema_without_unreferenced_defs def default_pt(messages): @@ -5949,11 +5949,11 @@ def _bedrock_tools_pt( else: description = name - remaining_inline_bytes = BEDROCK_TOOLS_MAX_INLINED_BYTES - inlined_bytes + remaining_inline_bytes = DEFS_MAX_INLINED_BYTES - inlined_bytes try: if resolves_schema_defs: tool_inlined_bytes = inline_legacy_defs(parameters, remaining_inline_bytes) - parameters = dict(schema_with_root_ref_resolved(parameters)) + parameters = dict(schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters))) else: defs = parameters.pop("$defs", {}) defs_copy = defs if isinstance(defs, dict) else {} @@ -5965,8 +5965,9 @@ def _bedrock_tools_pt( except ValueError as e: raise BedrockError( status_code=400, - message=f"Inlining the $ref definitions of tool '{raw_name}' pushes the request's tool schemas past " - f"{BEDROCK_TOOLS_MAX_INLINED_BYTES:,} bytes. Reference shared $defs less often or flatten the schema.", + message=f"Inlining the $ref definitions of tool '{raw_name}' copies more than " + f"{DEFS_MAX_INLINED_BYTES:,} bytes of definitions across the request's tools. " + "Reference shared $defs less often or flatten the schema.", ) from e inlined_bytes += tool_inlined_bytes normalize_json_schema_custom_types_to_object(parameters) diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py index 385684fe586..de00f02ea6c 100644 --- a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -5,6 +5,7 @@ from graphlib import CycleError, TopologicalSorter from itertools import chain, islice from types import MappingProxyType from typing import Final +from urllib.parse import unquote from pydantic import TypeAdapter @@ -13,6 +14,8 @@ _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") _LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) _MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 _EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) +_ROOT_DEFS_POINTER: Final = "#/$defs/" +_DYNAMIC_REF_KEYWORDS: Final = ("$dynamicRef", "$recursiveRef") def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: @@ -56,18 +59,76 @@ def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[s return target if isinstance(target, dict) else None +def _required_names_in_order(schema: Mapping[str, object]) -> tuple[str, ...]: + return tuple(name for name in _list_at(schema, "required") if isinstance(name, str)) + + def _with_root_ref_inlined(schema: Mapping[str, object]) -> Mapping[str, object]: ref: Final = schema.get("$ref") target: Final = _resolve_local_schema_ref(schema, ref) if isinstance(ref, str) else None if target is None: return schema - return {**target, **{key: value for key, value in schema.items() if key != "$ref"}} + siblings: Final = {key: value for key, value in schema.items() if key != "$ref"} + required: Final = dict.fromkeys((*_required_names_in_order(target), *_required_names_in_order(schema))) + return { + **target, + **siblings, + "properties": {**_schema_properties(target), **_schema_properties(schema)}, + "required": list(required), + } def schema_with_root_ref_resolved(schema: Mapping[str, object]) -> Mapping[str, object]: return reduce(lambda resolved, _hop: _with_root_ref_inlined(resolved), range(_MAX_SCHEMA_FLATTEN_DEPTH), schema) +def _referenced_def_names(ref: object) -> frozenset[str] | None: + if not isinstance(ref, str): + return frozenset() + pointer: Final = schema_fragment_ref(ref) + if pointer == "#" or (pointer.startswith("#/") and not pointer.startswith("#/$defs")): + return frozenset() + if not pointer.startswith(_ROOT_DEFS_POINTER): + return None + name: Final = pointer[len(_ROOT_DEFS_POINTER) :].split("/")[0] + return frozenset((name, unquote(name).replace("~1", "/").replace("~0", "~"))) + + +def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]: + if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS): + return (None,) + return (_referenced_def_names(node["$ref"]),) if "$ref" in node else () + + +def _schema_ref_def_names(schema: Mapping[str, object]) -> Iterator[frozenset[str] | None]: + return chain.from_iterable(_node_ref_def_names(node) for node in _schema_nodes(schema)) + + +def _reached_def_names(body: Mapping[str, object], defs: Mapping[str, object]) -> frozenset[str] | None: + reached: frozenset[str] = frozenset() # rebind-ok: grows by the definitions the last round's refs name + frontier: tuple[Mapping[str, object], ...] = (body,) # rebind-ok: the schemas the last round reached + while frontier: + found = tuple(chain.from_iterable(_schema_ref_def_names(node) for node in frontier)) + if None in found: + return None + new_names = frozenset(chain.from_iterable(names for names in found if names is not None)) - reached + reached = reached | new_names + frontier = tuple(definition for name in new_names if (definition := _schema_at(defs, name)) is not None) + return reached + + +def schema_without_unreferenced_defs(schema: Mapping[str, object]) -> Mapping[str, object]: + defs: Final = _schema_at(schema, "$defs") + if defs is None: + return schema + body: Final = {key: value for key, value in schema.items() if key != "$defs"} + reached: Final = _reached_def_names(body, defs) + if reached is None: + return schema + kept: Final = {name: definition for name, definition in defs.items() if name in reached} + return {**body, "$defs": kept} if kept else body + + def _mergeable_branch( root: Mapping[str, object], branch: object, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 27d967270f1..21d67cedf30 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2838,7 +2838,98 @@ def test_bedrock_tools_pt_follows_a_chain_of_root_refs_for_claude() -> None: sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] - assert sent == {**_ADDRESS_SCHEMA, "$defs": defs} + assert sent == _ADDRESS_SCHEMA + + +_ORDER_SCHEMA: Final = { + "type": "object", + "properties": {"street": {"type": "string"}, "city": {"type": "string"}}, + "required": ["street", "city"], +} + + +@pytest.mark.parametrize( + ("siblings", "expected"), + [ + ( + {"type": "object", "properties": {}, "additionalProperties": False}, + {**_ORDER_SCHEMA, "additionalProperties": False}, + ), + ( + { + "properties": {"city": {"type": "string", "enum": ["Paris"]}, "note": {"type": "string"}}, + "required": ["note"], + }, + { + "type": "object", + "properties": { + "street": {"type": "string"}, + "city": {"type": "string", "enum": ["Paris"]}, + "note": {"type": "string"}, + }, + "required": ["street", "city", "note"], + }, + ), + ], + ids=["mcp_normalized_empty_properties", "root_properties_win"], +) +def test_bedrock_tools_pt_merges_root_ref_siblings_into_its_target_for_claude( + siblings: dict[str, object], expected: dict[str, object] +) -> None: + parameters: Final = {"$ref": "#/$defs/Order", "$defs": {"Order": _ORDER_SCHEMA}, **siblings} + tools: Final = [{"type": "function", "function": {"name": "ship_order", "strict": True, "parameters": parameters}}] + + tool_spec: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-haiku-4-5-20251001-v1:0")[0]["toolSpec"] + + assert tool_spec["inputSchema"]["json"] == expected + + +def test_bedrock_tools_pt_sends_claude_only_the_defs_its_refs_reach() -> None: + defs: Final = { + "Home": {"type": "object", "properties": {"address": {"$ref": "#/$defs/Address"}}}, + "Address": _ADDRESS_SCHEMA, + "Place": _ORDER_SCHEMA, + "Unused": {"type": "string"}, + } + properties: Final = { + "home": {"$ref": "#/$defs/Home"}, + "city": {"$ref": "#/$defs/Place/properties/city"}, + "same_home": {"$ref": "#/properties/home"}, + "parent": {"$ref": "#"}, + } + parameters: Final = {"type": "object", "properties": properties, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == {"Home": defs["Home"], "Address": _ADDRESS_SCHEMA, "Place": _ORDER_SCHEMA} + + +def test_bedrock_tools_pt_keeps_a_claude_def_whose_name_its_ref_escapes() -> None: + defs: Final = {"a/b c~1": _ADDRESS_SCHEMA, "Unused": {"type": "string"}} + parameters: Final = {"type": "object", "properties": {"home": {"$ref": "#/$defs/a~1b%20c~01"}}, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == {"a/b c~1": _ADDRESS_SCHEMA} + + +@pytest.mark.parametrize( + "home", + [{"$ref": "#home"}, {"$dynamicRef": "#home"}, {"$ref": "#/$defs"}], + ids=["anchor_ref", "dynamic_ref", "whole_defs_ref"], +) +def test_bedrock_tools_pt_keeps_every_claude_def_when_a_ref_may_reach_one_by_other_means( + home: dict[str, object], +) -> None: + defs: Final = {"Home": {"$anchor": "home", **_ADDRESS_SCHEMA}, "Spare": {"type": "string"}} + parameters: Final = {"type": "object", "properties": {"home": home}, "$defs": defs} + tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}] + + sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] + + assert sent["$defs"] == defs def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() -> None: @@ -2850,8 +2941,7 @@ def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"] - home: Final = {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]} - assert sent == {**home, "$defs": {"Home": home}} + assert sent == {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]} def test_bedrock_tools_pt_inlines_only_legacy_refs_for_claude() -> None: diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6de831dd02e..3be84e83bbf 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -27,7 +27,7 @@ from litellm.llms.anthropic.chat.transformation import ( NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, NATIVE_OUTPUT_MAX_UNION_PARAMETERS, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import BEDROCK_TOOLS_MAX_INLINED_BYTES +from litellm.litellm_core_utils.prompt_templates.common_utils import DEFS_MAX_INLINED_BYTES from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES @@ -5787,8 +5787,21 @@ def test_tool_defs_that_double_at_every_level_fail_fast_with_a_400_naming_the_to mock_post.assert_not_called() +def test_tool_defs_that_inline_to_megabytes_still_reach_models_that_need_inlining(): + description: Final = "k" * 3_000_000 + parameters: Final = { + "type": "object", + "properties": {"key": {"$ref": "#/$defs/Key"}}, + "$defs": {"Key": {"type": "string", "description": description}}, + } + + sent: Final = _bedrock_tools_pt([_tool("lookup", parameters)], model="us.amazon.nova-2-lite-v1:0") + + assert sent[0]["toolSpec"]["inputSchema"]["json"]["properties"]["key"]["description"] == description + + def test_tool_defs_inlining_budget_covers_every_tool_in_the_request(): - description_bytes: Final = BEDROCK_TOOLS_MAX_INLINED_BYTES * 2 // 5 + description_bytes: Final = DEFS_MAX_INLINED_BYTES * 2 // 5 tools: Final = [ _tool( f"lookup_{index}",