mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): merge root ref siblings, drop unused $defs, and use the shared inlining budget
A root $ref beside an empty properties block, the shape MCP tools get, now merges into its target instead of replacing the target's properties, so a strict Claude tool sends its real arguments again. Claude also stops getting $defs entries no $ref reaches, which kept unused definitions out of its input tokens before this branch. Inlining for other models and for legacy definitions now uses the same 10 MB budget the native Anthropic route uses, since 1 MB refused an 852 KB expansion that worked at the merge base, and the 400 now names the bytes of definitions copied
This commit is contained in:
parent
75da9c820e
commit
6fc409da1a
5 changed files with 177 additions and 13 deletions
|
|
@ -1092,7 +1092,6 @@ def _legacy_defs(schema: object) -> Mapping[str, object] | None:
|
|||
# below memory-pressure territory, and rejects request-supplied bombs before
|
||||
# the proxy materialises them.
|
||||
DEFS_MAX_INLINED_BYTES: Final = 10_000_000
|
||||
BEDROCK_TOOLS_MAX_INLINED_BYTES: Final = 1_000_000
|
||||
|
||||
|
||||
def unpack_legacy_defs(
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ from litellm.types.llms.vertex_ai import PartType as VertexPartType
|
|||
from litellm.types.utils import GenericImageParsingChunk
|
||||
|
||||
from .common_utils import (
|
||||
BEDROCK_TOOLS_MAX_INLINED_BYTES,
|
||||
DEFS_MAX_INLINED_BYTES,
|
||||
as_openai_image_part,
|
||||
converse_redacted_content,
|
||||
convert_content_list_to_str,
|
||||
|
|
@ -81,7 +81,7 @@ from .common_utils import (
|
|||
unpack_defs,
|
||||
)
|
||||
from .image_handling import convert_url_to_base64, is_pdf_media_type
|
||||
from .json_schema_utils import schema_with_root_ref_resolved
|
||||
from .json_schema_utils import schema_with_root_ref_resolved, schema_without_unreferenced_defs
|
||||
|
||||
|
||||
def default_pt(messages):
|
||||
|
|
@ -5949,11 +5949,11 @@ def _bedrock_tools_pt(
|
|||
else:
|
||||
description = name
|
||||
|
||||
remaining_inline_bytes = BEDROCK_TOOLS_MAX_INLINED_BYTES - inlined_bytes
|
||||
remaining_inline_bytes = DEFS_MAX_INLINED_BYTES - inlined_bytes
|
||||
try:
|
||||
if resolves_schema_defs:
|
||||
tool_inlined_bytes = inline_legacy_defs(parameters, remaining_inline_bytes)
|
||||
parameters = dict(schema_with_root_ref_resolved(parameters))
|
||||
parameters = dict(schema_without_unreferenced_defs(schema_with_root_ref_resolved(parameters)))
|
||||
else:
|
||||
defs = parameters.pop("$defs", {})
|
||||
defs_copy = defs if isinstance(defs, dict) else {}
|
||||
|
|
@ -5965,8 +5965,9 @@ def _bedrock_tools_pt(
|
|||
except ValueError as e:
|
||||
raise BedrockError(
|
||||
status_code=400,
|
||||
message=f"Inlining the $ref definitions of tool '{raw_name}' pushes the request's tool schemas past "
|
||||
f"{BEDROCK_TOOLS_MAX_INLINED_BYTES:,} bytes. Reference shared $defs less often or flatten the schema.",
|
||||
message=f"Inlining the $ref definitions of tool '{raw_name}' copies more than "
|
||||
f"{DEFS_MAX_INLINED_BYTES:,} bytes of definitions across the request's tools. "
|
||||
"Reference shared $defs less often or flatten the schema.",
|
||||
) from e
|
||||
inlined_bytes += tool_inlined_bytes
|
||||
normalize_json_schema_custom_types_to_object(parameters)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from graphlib import CycleError, TopologicalSorter
|
|||
from itertools import chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import unquote
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
|
@ -13,6 +14,8 @@ _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not")
|
|||
_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions"))
|
||||
_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32
|
||||
_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_ROOT_DEFS_POINTER: Final = "#/$defs/"
|
||||
_DYNAMIC_REF_KEYWORDS: Final = ("$dynamicRef", "$recursiveRef")
|
||||
|
||||
|
||||
def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -56,18 +59,76 @@ def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[s
|
|||
return target if isinstance(target, dict) else None
|
||||
|
||||
|
||||
def _required_names_in_order(schema: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return tuple(name for name in _list_at(schema, "required") if isinstance(name, str))
|
||||
|
||||
|
||||
def _with_root_ref_inlined(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
ref: Final = schema.get("$ref")
|
||||
target: Final = _resolve_local_schema_ref(schema, ref) if isinstance(ref, str) else None
|
||||
if target is None:
|
||||
return schema
|
||||
return {**target, **{key: value for key, value in schema.items() if key != "$ref"}}
|
||||
siblings: Final = {key: value for key, value in schema.items() if key != "$ref"}
|
||||
required: Final = dict.fromkeys((*_required_names_in_order(target), *_required_names_in_order(schema)))
|
||||
return {
|
||||
**target,
|
||||
**siblings,
|
||||
"properties": {**_schema_properties(target), **_schema_properties(schema)},
|
||||
"required": list(required),
|
||||
}
|
||||
|
||||
|
||||
def schema_with_root_ref_resolved(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return reduce(lambda resolved, _hop: _with_root_ref_inlined(resolved), range(_MAX_SCHEMA_FLATTEN_DEPTH), schema)
|
||||
|
||||
|
||||
def _referenced_def_names(ref: object) -> frozenset[str] | None:
|
||||
if not isinstance(ref, str):
|
||||
return frozenset()
|
||||
pointer: Final = schema_fragment_ref(ref)
|
||||
if pointer == "#" or (pointer.startswith("#/") and not pointer.startswith("#/$defs")):
|
||||
return frozenset()
|
||||
if not pointer.startswith(_ROOT_DEFS_POINTER):
|
||||
return None
|
||||
name: Final = pointer[len(_ROOT_DEFS_POINTER) :].split("/")[0]
|
||||
return frozenset((name, unquote(name).replace("~1", "/").replace("~0", "~")))
|
||||
|
||||
|
||||
def _node_ref_def_names(node: Mapping[str, object]) -> tuple[frozenset[str] | None, ...]:
|
||||
if any(keyword in node for keyword in _DYNAMIC_REF_KEYWORDS):
|
||||
return (None,)
|
||||
return (_referenced_def_names(node["$ref"]),) if "$ref" in node else ()
|
||||
|
||||
|
||||
def _schema_ref_def_names(schema: Mapping[str, object]) -> Iterator[frozenset[str] | None]:
|
||||
return chain.from_iterable(_node_ref_def_names(node) for node in _schema_nodes(schema))
|
||||
|
||||
|
||||
def _reached_def_names(body: Mapping[str, object], defs: Mapping[str, object]) -> frozenset[str] | None:
|
||||
reached: frozenset[str] = frozenset() # rebind-ok: grows by the definitions the last round's refs name
|
||||
frontier: tuple[Mapping[str, object], ...] = (body,) # rebind-ok: the schemas the last round reached
|
||||
while frontier:
|
||||
found = tuple(chain.from_iterable(_schema_ref_def_names(node) for node in frontier))
|
||||
if None in found:
|
||||
return None
|
||||
new_names = frozenset(chain.from_iterable(names for names in found if names is not None)) - reached
|
||||
reached = reached | new_names
|
||||
frontier = tuple(definition for name in new_names if (definition := _schema_at(defs, name)) is not None)
|
||||
return reached
|
||||
|
||||
|
||||
def schema_without_unreferenced_defs(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
defs: Final = _schema_at(schema, "$defs")
|
||||
if defs is None:
|
||||
return schema
|
||||
body: Final = {key: value for key, value in schema.items() if key != "$defs"}
|
||||
reached: Final = _reached_def_names(body, defs)
|
||||
if reached is None:
|
||||
return schema
|
||||
kept: Final = {name: definition for name, definition in defs.items() if name in reached}
|
||||
return {**body, "$defs": kept} if kept else body
|
||||
|
||||
|
||||
def _mergeable_branch(
|
||||
root: Mapping[str, object],
|
||||
branch: object,
|
||||
|
|
|
|||
|
|
@ -2838,7 +2838,98 @@ def test_bedrock_tools_pt_follows_a_chain_of_root_refs_for_claude() -> None:
|
|||
|
||||
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
|
||||
|
||||
assert sent == {**_ADDRESS_SCHEMA, "$defs": defs}
|
||||
assert sent == _ADDRESS_SCHEMA
|
||||
|
||||
|
||||
_ORDER_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"street": {"type": "string"}, "city": {"type": "string"}},
|
||||
"required": ["street", "city"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("siblings", "expected"),
|
||||
[
|
||||
(
|
||||
{"type": "object", "properties": {}, "additionalProperties": False},
|
||||
{**_ORDER_SCHEMA, "additionalProperties": False},
|
||||
),
|
||||
(
|
||||
{
|
||||
"properties": {"city": {"type": "string", "enum": ["Paris"]}, "note": {"type": "string"}},
|
||||
"required": ["note"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"street": {"type": "string"},
|
||||
"city": {"type": "string", "enum": ["Paris"]},
|
||||
"note": {"type": "string"},
|
||||
},
|
||||
"required": ["street", "city", "note"],
|
||||
},
|
||||
),
|
||||
],
|
||||
ids=["mcp_normalized_empty_properties", "root_properties_win"],
|
||||
)
|
||||
def test_bedrock_tools_pt_merges_root_ref_siblings_into_its_target_for_claude(
|
||||
siblings: dict[str, object], expected: dict[str, object]
|
||||
) -> None:
|
||||
parameters: Final = {"$ref": "#/$defs/Order", "$defs": {"Order": _ORDER_SCHEMA}, **siblings}
|
||||
tools: Final = [{"type": "function", "function": {"name": "ship_order", "strict": True, "parameters": parameters}}]
|
||||
|
||||
tool_spec: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-haiku-4-5-20251001-v1:0")[0]["toolSpec"]
|
||||
|
||||
assert tool_spec["inputSchema"]["json"] == expected
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_sends_claude_only_the_defs_its_refs_reach() -> None:
|
||||
defs: Final = {
|
||||
"Home": {"type": "object", "properties": {"address": {"$ref": "#/$defs/Address"}}},
|
||||
"Address": _ADDRESS_SCHEMA,
|
||||
"Place": _ORDER_SCHEMA,
|
||||
"Unused": {"type": "string"},
|
||||
}
|
||||
properties: Final = {
|
||||
"home": {"$ref": "#/$defs/Home"},
|
||||
"city": {"$ref": "#/$defs/Place/properties/city"},
|
||||
"same_home": {"$ref": "#/properties/home"},
|
||||
"parent": {"$ref": "#"},
|
||||
}
|
||||
parameters: Final = {"type": "object", "properties": properties, "$defs": defs}
|
||||
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
|
||||
|
||||
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
|
||||
|
||||
assert sent["$defs"] == {"Home": defs["Home"], "Address": _ADDRESS_SCHEMA, "Place": _ORDER_SCHEMA}
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_keeps_a_claude_def_whose_name_its_ref_escapes() -> None:
|
||||
defs: Final = {"a/b c~1": _ADDRESS_SCHEMA, "Unused": {"type": "string"}}
|
||||
parameters: Final = {"type": "object", "properties": {"home": {"$ref": "#/$defs/a~1b%20c~01"}}, "$defs": defs}
|
||||
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
|
||||
|
||||
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
|
||||
|
||||
assert sent["$defs"] == {"a/b c~1": _ADDRESS_SCHEMA}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"home",
|
||||
[{"$ref": "#home"}, {"$dynamicRef": "#home"}, {"$ref": "#/$defs"}],
|
||||
ids=["anchor_ref", "dynamic_ref", "whole_defs_ref"],
|
||||
)
|
||||
def test_bedrock_tools_pt_keeps_every_claude_def_when_a_ref_may_reach_one_by_other_means(
|
||||
home: dict[str, object],
|
||||
) -> None:
|
||||
defs: Final = {"Home": {"$anchor": "home", **_ADDRESS_SCHEMA}, "Spare": {"type": "string"}}
|
||||
parameters: Final = {"type": "object", "properties": {"home": home}, "$defs": defs}
|
||||
tools: Final = [{"type": "function", "function": {"name": "save_home", "parameters": parameters}}]
|
||||
|
||||
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
|
||||
|
||||
assert sent["$defs"] == defs
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude() -> None:
|
||||
|
|
@ -2850,8 +2941,7 @@ def test_bedrock_tools_pt_inlines_legacy_refs_under_a_root_defs_ref_for_claude()
|
|||
|
||||
sent: Final = _bedrock_tools_pt(tools, model="us.anthropic.claude-sonnet-5-5")[0]["toolSpec"]["inputSchema"]["json"]
|
||||
|
||||
home: Final = {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]}
|
||||
assert sent == {**home, "$defs": {"Home": home}}
|
||||
assert sent == {"type": "object", "properties": {"address": _ADDRESS_SCHEMA}, "required": ["address"]}
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_inlines_only_legacy_refs_for_claude() -> None:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from litellm.llms.anthropic.chat.transformation import (
|
|||
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS,
|
||||
NATIVE_OUTPUT_MAX_UNION_PARAMETERS,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import BEDROCK_TOOLS_MAX_INLINED_BYTES
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import DEFS_MAX_INLINED_BYTES
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
|
||||
|
|
@ -5787,8 +5787,21 @@ def test_tool_defs_that_double_at_every_level_fail_fast_with_a_400_naming_the_to
|
|||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
def test_tool_defs_that_inline_to_megabytes_still_reach_models_that_need_inlining():
|
||||
description: Final = "k" * 3_000_000
|
||||
parameters: Final = {
|
||||
"type": "object",
|
||||
"properties": {"key": {"$ref": "#/$defs/Key"}},
|
||||
"$defs": {"Key": {"type": "string", "description": description}},
|
||||
}
|
||||
|
||||
sent: Final = _bedrock_tools_pt([_tool("lookup", parameters)], model="us.amazon.nova-2-lite-v1:0")
|
||||
|
||||
assert sent[0]["toolSpec"]["inputSchema"]["json"]["properties"]["key"]["description"] == description
|
||||
|
||||
|
||||
def test_tool_defs_inlining_budget_covers_every_tool_in_the_request():
|
||||
description_bytes: Final = BEDROCK_TOOLS_MAX_INLINED_BYTES * 2 // 5
|
||||
description_bytes: Final = DEFS_MAX_INLINED_BYTES * 2 // 5
|
||||
tools: Final = [
|
||||
_tool(
|
||||
f"lookup_{index}",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue